Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.Message
  20 import email.Generator
  21 import rfc822
  22 import StringIO
  23 try:
  24     if sys.platform == 'os2emx':
  25         # OS/2 EMX fcntl() not adequate
  26         raise ImportError
  27     import fcntl
  28 except ImportError:
  29     fcntl = None
  30
  31 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  32             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  33             'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  34             'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  35
  36 class Mailbox:
  37     """A group of messages in a particular place."""
  38
  39     def __init__(self, path, factory=None, create=True):
  40         """Initialize a Mailbox instance."""
  41         self._path = os.path.abspath(os.path.expanduser(path))
  42         self._factory = factory
  43
  44     def add(self, message):
  45         """Add message and return assigned key."""
  46         raise NotImplementedError('Method must be implemented by subclass')
  47
  48     def remove(self, key):
  49         """Remove the keyed message; raise KeyError if it doesn't exist."""
  50         raise NotImplementedError('Method must be implemented by subclass')
  51
  52     def __delitem__(self, key):
  53         self.remove(key)
  54
  55     def discard(self, key):
  56         """If the keyed message exists, remove it."""
  57         try:
  58             self.remove(key)
  59         except KeyError:
  60             pass
  61
  62     def __setitem__(self, key, message):
  63         """Replace the keyed message; raise KeyError if it doesn't exist."""
  64         raise NotImplementedError('Method must be implemented by subclass')
  65
  66     def get(self, key, default=None):
  67         """Return the keyed message, or default if it doesn't exist."""
  68         try:
  69             return self.__getitem__(key)
  70         except KeyError:
  71             return default
  72
  73     def __getitem__(self, key):
  74         """Return the keyed message; raise KeyError if it doesn't exist."""
  75         if not self._factory:
  76             return self.get_message(key)
  77         else:
  78             return self._factory(self.get_file(key))
  79
  80     def get_message(self, key):
  81         """Return a Message representation or raise a KeyError."""
  82         raise NotImplementedError('Method must be implemented by subclass')
  83
  84     def get_string(self, key):
  85         """Return a string representation or raise a KeyError."""
  86         raise NotImplementedError('Method must be implemented by subclass')
  87
  88     def get_file(self, key):
  89         """Return a file-like representation or raise a KeyError."""
  90         raise NotImplementedError('Method must be implemented by subclass')
  91
  92     def iterkeys(self):
  93         """Return an iterator over keys."""
  94         raise NotImplementedError('Method must be implemented by subclass')
  95
  96     def keys(self):
  97         """Return a list of keys."""
  98         return list(self.iterkeys())
  99
 100     def itervalues(self):
 101         """Return an iterator over all messages."""
 102         for key in self.iterkeys():
 103             try:
 104                 value = self[key]
 105             except KeyError:
 106                 continue
 107             yield value
 108
 109     def __iter__(self):
 110         return self.itervalues()
 111
 112     def values(self):
 113         """Return a list of messages. Memory intensive."""
 114         return list(self.itervalues())
 115
 116     def iteritems(self):
 117         """Return an iterator over (key, message) tuples."""
 118         for key in self.iterkeys():
 119             try:
 120                 value = self[key]
 121             except KeyError:
 122                 continue
 123             yield (key, value)
 124
 125     def items(self):
 126         """Return a list of (key, message) tuples. Memory intensive."""
 127         return list(self.iteritems())
 128
 129     def has_key(self, key):
 130         """Return True if the keyed message exists, False otherwise."""
 131         raise NotImplementedError('Method must be implemented by subclass')
 132
 133     def __contains__(self, key):
 134         return self.has_key(key)
 135
 136     def __len__(self):
 137         """Return a count of messages in the mailbox."""
 138         raise NotImplementedError('Method must be implemented by subclass')
 139
 140     def clear(self):
 141         """Delete all messages."""
 142         for key in self.iterkeys():
 143             self.discard(key)
 144
 145     def pop(self, key, default=None):
 146         """Delete the keyed message and return it, or default."""
 147         try:
 148             result = self[key]
 149         except KeyError:
 150             return default
 151         self.discard(key)
 152         return result
 153
 154     def popitem(self):
 155         """Delete an arbitrary (key, message) pair and return it."""
 156         for key in self.iterkeys():
 157             return (key, self.pop(key))     # This is only run once.
 158         else:
 159             raise KeyError('No messages in mailbox')
 160
 161     def update(self, arg=None):
 162         """Change the messages that correspond to certain keys."""
 163         if hasattr(arg, 'iteritems'):
 164             source = arg.iteritems()
 165         elif hasattr(arg, 'items'):
 166             source = arg.items()
 167         else:
 168             source = arg
 169         bad_key = False
 170         for key, message in source:
 171             try:
 172                 self[key] = message
 173             except KeyError:
 174                 bad_key = True
 175         if bad_key:
 176             raise KeyError('No message with key(s)')
 177
 178     def flush(self):
 179         """Write any pending changes to the disk."""
 180         raise NotImplementedError('Method must be implemented by subclass')
 181
 182     def lock(self):
 183         """Lock the mailbox."""
 184         raise NotImplementedError('Method must be implemented by subclass')
 185
 186     def unlock(self):
 187         """Unlock the mailbox if it is locked."""
 188         raise NotImplementedError('Method must be implemented by subclass')
 189
 190     def close(self):
 191         """Flush and close the mailbox."""
 192         raise NotImplementedError('Method must be implemented by subclass')
 193
 194     def _dump_message(self, message, target, mangle_from_=False):
 195         # Most files are opened in binary mode to allow predictable seeking.
 196         # To get native line endings on disk, the user-friendly \n line endings
 197         # used in strings and by email.Message are translated here.
 198         """Dump message contents to target file."""
 199         if isinstance(message, email.Message.Message):
 200             buffer = StringIO.StringIO()
 201             gen = email.Generator.Generator(buffer, mangle_from_, 0)
 202             gen.flatten(message)
 203             buffer.seek(0)
 204             target.write(buffer.read().replace('\n', os.linesep))
 205         elif isinstance(message, str):
 206             if mangle_from_:
 207                 message = message.replace('\nFrom ', '\n>From ')
 208             message = message.replace('\n', os.linesep)
 209             target.write(message)
 210         elif hasattr(message, 'read'):
 211             while True:
 212                 line = message.readline()
 213                 if line == '':
 214                     break
 215                 if mangle_from_ and line.startswith('From '):
 216                     line = '>From ' + line[5:]
 217                 line = line.replace('\n', os.linesep)
 218                 target.write(line)
 219         else:
 220             raise TypeError('Invalid message type: %s' % type(message))
 221
 222
 223 class Maildir(Mailbox):
 224     """A qmail-style Maildir mailbox."""
 225
 226     colon = ':'
 227
 228     def __init__(self, dirname, factory=rfc822.Message, create=True):
 229         """Initialize a Maildir instance."""
 230         Mailbox.__init__(self, dirname, factory, create)
 231         if not os.path.exists(self._path):
 232             if create:
 233                 os.mkdir(self._path, 0700)
 234                 os.mkdir(os.path.join(self._path, 'tmp'), 0700)
 235                 os.mkdir(os.path.join(self._path, 'new'), 0700)
 236                 os.mkdir(os.path.join(self._path, 'cur'), 0700)
 237             else:
 238                 raise NoSuchMailboxError(self._path)
 239         self._toc = {}
 240
 241     def add(self, message):
 242         """Add message and return assigned key."""
 243         tmp_file = self._create_tmp()
 244         try:
 245             self._dump_message(message, tmp_file)
 246         finally:
 247             _sync_close(tmp_file)
 248         if isinstance(message, MaildirMessage):
 249             subdir = message.get_subdir()
 250             suffix = self.colon + message.get_info()
 251             if suffix == self.colon:
 252                 suffix = ''
 253         else:
 254             subdir = 'new'
 255             suffix = ''
 256         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 257         dest = os.path.join(self._path, subdir, uniq + suffix)
 258         try:
 259             if hasattr(os, 'link'):
 260                 os.link(tmp_file.name, dest)
 261                 os.remove(tmp_file.name)
 262             else:
 263                 os.rename(tmp_file.name, dest)
 264         except OSError, e:
 265             os.remove(tmp_file.name)
 266             if e.errno == errno.EEXIST:
 267                 raise ExternalClashError('Name clash with existing message: %s'
 268                                          % dest)
 269             else:
 270                 raise
 271         if isinstance(message, MaildirMessage):
 272             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 273         return uniq
 274
 275     def remove(self, key):
 276         """Remove the keyed message; raise KeyError if it doesn't exist."""
 277         os.remove(os.path.join(self._path, self._lookup(key)))
 278
 279     def discard(self, key):
 280         """If the keyed message exists, remove it."""
 281         # This overrides an inapplicable implementation in the superclass.
 282         try:
 283             self.remove(key)
 284         except KeyError:
 285             pass
 286         except OSError, e:
 287             if e.errno != errno.ENOENT:
 288                 raise
 289
 290     def __setitem__(self, key, message):
 291         """Replace the keyed message; raise KeyError if it doesn't exist."""
 292         old_subpath = self._lookup(key)
 293         temp_key = self.add(message)
 294         temp_subpath = self._lookup(temp_key)
 295         if isinstance(message, MaildirMessage):
 296             # temp's subdir and suffix were specified by message.
 297             dominant_subpath = temp_subpath
 298         else:
 299             # temp's subdir and suffix were defaults from add().
 300             dominant_subpath = old_subpath
 301         subdir = os.path.dirname(dominant_subpath)
 302         if self.colon in dominant_subpath:
 303             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 304         else:
 305             suffix = ''
 306         self.discard(key)
 307         new_path = os.path.join(self._path, subdir, key + suffix)
 308         os.rename(os.path.join(self._path, temp_subpath), new_path)
 309         if isinstance(message, MaildirMessage):
 310             os.utime(new_path, (os.path.getatime(new_path),
 311                                 message.get_date()))
 312
 313     def get_message(self, key):
 314         """Return a Message representation or raise a KeyError."""
 315         subpath = self._lookup(key)
 316         f = open(os.path.join(self._path, subpath), 'r')
 317         try:
 318             msg = MaildirMessage(f)
 319         finally:
 320             f.close()
 321         subdir, name = os.path.split(subpath)
 322         msg.set_subdir(subdir)
 323         if self.colon in name:
 324             msg.set_info(name.split(self.colon)[-1])
 325         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 326         return msg
 327
 328     def get_string(self, key):
 329         """Return a string representation or raise a KeyError."""
 330         f = open(os.path.join(self._path, self._lookup(key)), 'r')
 331         try:
 332             return f.read()
 333         finally:
 334             f.close()
 335
 336     def get_file(self, key):
 337         """Return a file-like representation or raise a KeyError."""
 338         f = open(os.path.join(self._path, self._lookup(key)), 'rb')
 339         return _ProxyFile(f)
 340
 341     def iterkeys(self):
 342         """Return an iterator over keys."""
 343         self._refresh()
 344         for key in self._toc:
 345             try:
 346                 self._lookup(key)
 347             except KeyError:
 348                 continue
 349             yield key
 350
 351     def has_key(self, key):
 352         """Return True if the keyed message exists, False otherwise."""
 353         self._refresh()
 354         return key in self._toc
 355
 356     def __len__(self):
 357         """Return a count of messages in the mailbox."""
 358         self._refresh()
 359         return len(self._toc)
 360
 361     def flush(self):
 362         """Write any pending changes to disk."""
 363         return  # Maildir changes are always written immediately.
 364
 365     def lock(self):
 366         """Lock the mailbox."""
 367         return
 368
 369     def unlock(self):
 370         """Unlock the mailbox if it is locked."""
 371         return
 372
 373     def close(self):
 374         """Flush and close the mailbox."""
 375         return
 376
 377     def list_folders(self):
 378         """Return a list of folder names."""
 379         result = []
 380         for entry in os.listdir(self._path):
 381             if len(entry) > 1 and entry[0] == '.' and \
 382                os.path.isdir(os.path.join(self._path, entry)):
 383                 result.append(entry[1:])
 384         return result
 385
 386     def get_folder(self, folder):
 387         """Return a Maildir instance for the named folder."""
 388         return Maildir(os.path.join(self._path, '.' + folder),
 389                        factory=self._factory,
 390                        create=False)
 391
 392     def add_folder(self, folder):
 393         """Create a folder and return a Maildir instance representing it."""
 394         path = os.path.join(self._path, '.' + folder)
 395         result = Maildir(path, factory=self._factory)
 396         maildirfolder_path = os.path.join(path, 'maildirfolder')
 397         if not os.path.exists(maildirfolder_path):
 398             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY))
 399         return result
 400
 401     def remove_folder(self, folder):
 402         """Delete the named folder, which must be empty."""
 403         path = os.path.join(self._path, '.' + folder)
 404         for entry in os.listdir(os.path.join(path, 'new')) + \
 405                      os.listdir(os.path.join(path, 'cur')):
 406             if len(entry) < 1 or entry[0] != '.':
 407                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 408         for entry in os.listdir(path):
 409             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 410                os.path.isdir(os.path.join(path, entry)):
 411                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 412                                     (folder, entry))
 413         for root, dirs, files in os.walk(path, topdown=False):
 414             for entry in files:
 415                 os.remove(os.path.join(root, entry))
 416             for entry in dirs:
 417                 os.rmdir(os.path.join(root, entry))
 418         os.rmdir(path)
 419
 420     def clean(self):
 421         """Delete old files in "tmp"."""
 422         now = time.time()
 423         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 424             path = os.path.join(self._path, 'tmp', entry)
 425             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 426                 os.remove(path)
 427
 428     _count = 1  # This is used to generate unique file names.
 429
 430     def _create_tmp(self):
 431         """Create a file in the tmp subdirectory and open and return it."""
 432         now = time.time()
 433         hostname = socket.gethostname()
 434         if '/' in hostname:
 435             hostname = hostname.replace('/', r'\057')
 436         if ':' in hostname:
 437             hostname = hostname.replace(':', r'\072')
 438         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 439                                     Maildir._count, hostname)
 440         path = os.path.join(self._path, 'tmp', uniq)
 441         try:
 442             os.stat(path)
 443         except OSError, e:
 444             if e.errno == errno.ENOENT:
 445                 Maildir._count += 1
 446                 try:
 447                     return _create_carefully(path)
 448                 except OSError, e:
 449                     if e.errno != errno.EEXIST:
 450                         raise
 451             else:
 452                 raise
 453
 454         # Fall through to here if stat succeeded or open raised EEXIST.
 455         raise ExternalClashError('Name clash prevented file creation: %s' %
 456                                  path)
 457
 458     def _refresh(self):
 459         """Update table of contents mapping."""
 460         self._toc = {}
 461         for subdir in ('new', 'cur'):
 462             for entry in os.listdir(os.path.join(self._path, subdir)):
 463                 uniq = entry.split(self.colon)[0]
 464                 self._toc[uniq] = os.path.join(subdir, entry)
 465
 466     def _lookup(self, key):
 467         """Use TOC to return subpath for given key, or raise a KeyError."""
 468         try:
 469             if os.path.exists(os.path.join(self._path, self._toc[key])):
 470                 return self._toc[key]
 471         except KeyError:
 472             pass
 473         self._refresh()
 474         try:
 475             return self._toc[key]
 476         except KeyError:
 477             raise KeyError('No message with key: %s' % key)
 478
 479     # This method is for backward compatibility only.
 480     def next(self):
 481         """Return the next message in a one-time iteration."""
 482         if not hasattr(self, '_onetime_keys'):
 483             self._onetime_keys = self.iterkeys()
 484         while True:
 485             try:
 486                 return self[self._onetime_keys.next()]
 487             except StopIteration:
 488                 return None
 489             except KeyError:
 490                 continue
 491
 492
 493 class _singlefileMailbox(Mailbox):
 494     """A single-file mailbox."""
 495
 496     def __init__(self, path, factory=None, create=True):
 497         """Initialize a single-file mailbox."""
 498         Mailbox.__init__(self, path, factory, create)
 499         try:
 500             f = open(self._path, 'rb+')
 501         except IOError, e:
 502             if e.errno == errno.ENOENT:
 503                 if create:
 504                     f = open(self._path, 'wb+')
 505                 else:
 506                     raise NoSuchMailboxError(self._path)
 507             elif e.errno == errno.EACCES:
 508                 f = open(self._path, 'rb')
 509             else:
 510                 raise
 511         self._file = f
 512         self._toc = None
 513         self._next_key = 0
 514         self._pending = False   # No changes require rewriting the file.
 515         self._locked = False
 516
 517     def add(self, message):
 518         """Add message and return assigned key."""
 519         self._lookup()
 520         self._toc[self._next_key] = self._append_message(message)
 521         self._next_key += 1
 522         self._pending = True
 523         return self._next_key - 1
 524
 525     def remove(self, key):
 526         """Remove the keyed message; raise KeyError if it doesn't exist."""
 527         self._lookup(key)
 528         del self._toc[key]
 529         self._pending = True
 530
 531     def __setitem__(self, key, message):
 532         """Replace the keyed message; raise KeyError if it doesn't exist."""
 533         self._lookup(key)
 534         self._toc[key] = self._append_message(message)
 535         self._pending = True
 536
 537     def iterkeys(self):
 538         """Return an iterator over keys."""
 539         self._lookup()
 540         for key in self._toc.keys():
 541             yield key
 542
 543     def has_key(self, key):
 544         """Return True if the keyed message exists, False otherwise."""
 545         self._lookup()
 546         return key in self._toc
 547
 548     def __len__(self):
 549         """Return a count of messages in the mailbox."""
 550         self._lookup()
 551         return len(self._toc)
 552
 553     def lock(self):
 554         """Lock the mailbox."""
 555         if not self._locked:
 556             _lock_file(self._file)
 557             self._locked = True
 558
 559     def unlock(self):
 560         """Unlock the mailbox if it is locked."""
 561         if self._locked:
 562             _unlock_file(self._file)
 563             self._locked = False
 564
 565     def flush(self):
 566         """Write any pending changes to disk."""
 567         if not self._pending:
 568             return
 569         self._lookup()
 570         new_file = _create_temporary(self._path)
 571         try:
 572             new_toc = {}
 573             self._pre_mailbox_hook(new_file)
 574             for key in sorted(self._toc.keys()):
 575                 start, stop = self._toc[key]
 576                 self._file.seek(start)
 577                 self._pre_message_hook(new_file)
 578                 new_start = new_file.tell()
 579                 while True:
 580                     buffer = self._file.read(min(4096,
 581                                                  stop - self._file.tell()))
 582                     if buffer == '':
 583                         break
 584                     new_file.write(buffer)
 585                 new_toc[key] = (new_start, new_file.tell())
 586                 self._post_message_hook(new_file)
 587         except:
 588             new_file.close()
 589             os.remove(new_file.name)
 590             raise
 591         _sync_close(new_file)
 592         # self._file is about to get replaced, so no need to sync.
 593         self._file.close()
 594         try:
 595             os.rename(new_file.name, self._path)
 596         except OSError, e:
 597             if e.errno == errno.EEXIST or \
 598               (os.name == 'os2' and e.errno == errno.EACCES):
 599                 os.remove(self._path)
 600                 os.rename(new_file.name, self._path)
 601             else:
 602                 raise
 603         self._file = open(self._path, 'rb+')
 604         self._toc = new_toc
 605         self._pending = False
 606         if self._locked:
 607             _lock_file(self._file, dotlock=False)
 608
 609     def _pre_mailbox_hook(self, f):
 610         """Called before writing the mailbox to file f."""
 611         return
 612
 613     def _pre_message_hook(self, f):
 614         """Called before writing each message to file f."""
 615         return
 616
 617     def _post_message_hook(self, f):
 618         """Called after writing each message to file f."""
 619         return
 620
 621     def close(self):
 622         """Flush and close the mailbox."""
 623         self.flush()
 624         if self._locked:
 625             self.unlock()
 626         self._file.close()  # Sync has been done by self.flush() above.
 627
 628     def _lookup(self, key=None):
 629         """Return (start, stop) or raise KeyError."""
 630         if self._toc is None:
 631             self._generate_toc()
 632         if key is not None:
 633             try:
 634                 return self._toc[key]
 635             except KeyError:
 636                 raise KeyError('No message with key: %s' % key)
 637
 638     def _append_message(self, message):
 639         """Append message to mailbox and return (start, stop) offsets."""
 640         self._file.seek(0, 2)
 641         self._pre_message_hook(self._file)
 642         offsets = self._install_message(message)
 643         self._post_message_hook(self._file)
 644         self._file.flush()
 645         return offsets
 646
 647
 648
 649 class _mboxMMDF(_singlefileMailbox):
 650     """An mbox or MMDF mailbox."""
 651
 652     _mangle_from_ = True
 653
 654     def get_message(self, key):
 655         """Return a Message representation or raise a KeyError."""
 656         start, stop = self._lookup(key)
 657         self._file.seek(start)
 658         from_line = self._file.readline().replace(os.linesep, '')
 659         string = self._file.read(stop - self._file.tell())
 660         msg = self._message_factory(string.replace(os.linesep, '\n'))
 661         msg.set_from(from_line[5:])
 662         return msg
 663
 664     def get_string(self, key, from_=False):
 665         """Return a string representation or raise a KeyError."""
 666         start, stop = self._lookup(key)
 667         self._file.seek(start)
 668         if not from_:
 669             self._file.readline()
 670         string = self._file.read(stop - self._file.tell())
 671         return string.replace(os.linesep, '\n')
 672
 673     def get_file(self, key, from_=False):
 674         """Return a file-like representation or raise a KeyError."""
 675         start, stop = self._lookup(key)
 676         self._file.seek(start)
 677         if not from_:
 678             self._file.readline()
 679         return _PartialFile(self._file, self._file.tell(), stop)
 680
 681     def _install_message(self, message):
 682         """Format a message and blindly write to self._file."""
 683         from_line = None
 684         if isinstance(message, str) and message.startswith('From '):
 685             newline = message.find('\n')
 686             if newline != -1:
 687                 from_line = message[:newline]
 688                 message = message[newline + 1:]
 689             else:
 690                 from_line = message
 691                 message = ''
 692         elif isinstance(message, _mboxMMDFMessage):
 693             from_line = 'From ' + message.get_from()
 694         elif isinstance(message, email.Message.Message):
 695             from_line = message.get_unixfrom()  # May be None.
 696         if from_line is None:
 697             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 698         start = self._file.tell()
 699         self._file.write(from_line + os.linesep)
 700         self._dump_message(message, self._file, self._mangle_from_)
 701         stop = self._file.tell()
 702         return (start, stop)
 703
 704
 705 class mbox(_mboxMMDF):
 706     """A classic mbox mailbox."""
 707
 708     _mangle_from_ = True
 709
 710     def __init__(self, path, factory=None, create=True):
 711         """Initialize an mbox mailbox."""
 712         self._message_factory = mboxMessage
 713         _mboxMMDF.__init__(self, path, factory, create)
 714
 715     def _pre_message_hook(self, f):
 716         """Called before writing each message to file f."""
 717         if f.tell() != 0:
 718             f.write(os.linesep)
 719
 720     def _generate_toc(self):
 721         """Generate key-to-(start, stop) table of contents."""
 722         starts, stops = [], []
 723         self._file.seek(0)
 724         while True:
 725             line_pos = self._file.tell()
 726             line = self._file.readline()
 727             if line.startswith('From '):
 728                 if len(stops) < len(starts):
 729                     stops.append(line_pos - len(os.linesep))
 730                 starts.append(line_pos)
 731             elif line == '':
 732                 stops.append(line_pos)
 733                 break
 734         self._toc = dict(enumerate(zip(starts, stops)))
 735         self._next_key = len(self._toc)
 736
 737
 738 class MMDF(_mboxMMDF):
 739     """An MMDF mailbox."""
 740
 741     def __init__(self, path, factory=None, create=True):
 742         """Initialize an MMDF mailbox."""
 743         self._message_factory = MMDFMessage
 744         _mboxMMDF.__init__(self, path, factory, create)
 745
 746     def _pre_message_hook(self, f):
 747         """Called before writing each message to file f."""
 748         f.write('\001\001\001\001' + os.linesep)
 749
 750     def _post_message_hook(self, f):
 751         """Called after writing each message to file f."""
 752         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 753
 754     def _generate_toc(self):
 755         """Generate key-to-(start, stop) table of contents."""
 756         starts, stops = [], []
 757         self._file.seek(0)
 758         next_pos = 0
 759         while True:
 760             line_pos = next_pos
 761             line = self._file.readline()
 762             next_pos = self._file.tell()
 763             if line.startswith('\001\001\001\001' + os.linesep):
 764                 starts.append(next_pos)
 765                 while True:
 766                     line_pos = next_pos
 767                     line = self._file.readline()
 768                     next_pos = self._file.tell()
 769                     if line == '\001\001\001\001' + os.linesep:
 770                         stops.append(line_pos - len(os.linesep))
 771                         break
 772                     elif line == '':
 773                         stops.append(line_pos)
 774                         break
 775             elif line == '':
 776                 break
 777         self._toc = dict(enumerate(zip(starts, stops)))
 778         self._next_key = len(self._toc)
 779
 780
 781 class MH(Mailbox):
 782     """An MH mailbox."""
 783
 784     def __init__(self, path, factory=None, create=True):
 785         """Initialize an MH instance."""
 786         Mailbox.__init__(self, path, factory, create)
 787         if not os.path.exists(self._path):
 788             if create:
 789                 os.mkdir(self._path, 0700)
 790                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 791                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
 792             else:
 793                 raise NoSuchMailboxError(self._path)
 794         self._locked = False
 795
 796     def add(self, message):
 797         """Add message and return assigned key."""
 798         keys = self.keys()
 799         if len(keys) == 0:
 800             new_key = 1
 801         else:
 802             new_key = max(keys) + 1
 803         new_path = os.path.join(self._path, str(new_key))
 804         f = _create_carefully(new_path)
 805         try:
 806             if self._locked:
 807                 _lock_file(f)
 808             try:
 809                 self._dump_message(message, f)
 810                 if isinstance(message, MHMessage):
 811                     self._dump_sequences(message, new_key)
 812             finally:
 813                 if self._locked:
 814                     _unlock_file(f)
 815         finally:
 816             _sync_close(f)
 817         return new_key
 818
 819     def remove(self, key):
 820         """Remove the keyed message; raise KeyError if it doesn't exist."""
 821         path = os.path.join(self._path, str(key))
 822         try:
 823             f = open(path, 'rb+')
 824         except IOError, e:
 825             if e.errno == errno.ENOENT:
 826                 raise KeyError('No message with key: %s' % key)
 827             else:
 828                 raise
 829         try:
 830             if self._locked:
 831                 _lock_file(f)
 832             try:
 833                 f.close()
 834                 os.remove(os.path.join(self._path, str(key)))
 835             finally:
 836                 if self._locked:
 837                     _unlock_file(f)
 838         finally:
 839             f.close()
 840
 841     def __setitem__(self, key, message):
 842         """Replace the keyed message; raise KeyError if it doesn't exist."""
 843         path = os.path.join(self._path, str(key))
 844         try:
 845             f = open(path, 'rb+')
 846         except IOError, e:
 847             if e.errno == errno.ENOENT:
 848                 raise KeyError('No message with key: %s' % key)
 849             else:
 850                 raise
 851         try:
 852             if self._locked:
 853                 _lock_file(f)
 854             try:
 855                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 856                 self._dump_message(message, f)
 857                 if isinstance(message, MHMessage):
 858                     self._dump_sequences(message, key)
 859             finally:
 860                 if self._locked:
 861                     _unlock_file(f)
 862         finally:
 863             _sync_close(f)
 864
 865     def get_message(self, key):
 866         """Return a Message representation or raise a KeyError."""
 867         try:
 868             if self._locked:
 869                 f = open(os.path.join(self._path, str(key)), 'r+')
 870             else:
 871                 f = open(os.path.join(self._path, str(key)), 'r')
 872         except IOError, e:
 873             if e.errno == errno.ENOENT:
 874                 raise KeyError('No message with key: %s' % key)
 875             else:
 876                 raise
 877         try:
 878             if self._locked:
 879                 _lock_file(f)
 880             try:
 881                 msg = MHMessage(f)
 882             finally:
 883                 if self._locked:
 884                     _unlock_file(f)
 885         finally:
 886             f.close()
 887         for name, key_list in self.get_sequences():
 888             if key in key_list:
 889                 msg.add_sequence(name)
 890         return msg
 891
 892     def get_string(self, key):
 893         """Return a string representation or raise a KeyError."""
 894         try:
 895             if self._locked:
 896                 f = open(os.path.join(self._path, str(key)), 'r+')
 897             else:
 898                 f = open(os.path.join(self._path, str(key)), 'r')
 899         except IOError, e:
 900             if e.errno == errno.ENOENT:
 901                 raise KeyError('No message with key: %s' % key)
 902             else:
 903                 raise
 904         try:
 905             if self._locked:
 906                 _lock_file(f)
 907             try:
 908                 return f.read()
 909             finally:
 910                 if self._locked:
 911                     _unlock_file(f)
 912         finally:
 913             f.close()
 914
 915     def get_file(self, key):
 916         """Return a file-like representation or raise a KeyError."""
 917         try:
 918             f = open(os.path.join(self._path, str(key)), 'rb')
 919         except IOError, e:
 920             if e.errno == errno.ENOENT:
 921                 raise KeyError('No message with key: %s' % key)
 922             else:
 923                 raise
 924         return _ProxyFile(f)
 925
 926     def iterkeys(self):
 927         """Return an iterator over keys."""
 928         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 929                                       if entry.isdigit()))
 930
 931     def has_key(self, key):
 932         """Return True if the keyed message exists, False otherwise."""
 933         return os.path.exists(os.path.join(self._path, str(key)))
 934
 935     def __len__(self):
 936         """Return a count of messages in the mailbox."""
 937         return len(list(self.iterkeys()))
 938
 939     def lock(self):
 940         """Lock the mailbox."""
 941         if not self._locked:
 942             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
 943             _lock_file(self._file)
 944             self._locked = True
 945
 946     def unlock(self):
 947         """Unlock the mailbox if it is locked."""
 948         if self._locked:
 949             _unlock_file(self._file)
 950             _sync_close(self._file)
 951             del self._file
 952             self._locked = False
 953
 954     def flush(self):
 955         """Write any pending changes to the disk."""
 956         return
 957
 958     def close(self):
 959         """Flush and close the mailbox."""
 960         if self._locked:
 961             self.unlock()
 962
 963     def list_folders(self):
 964         """Return a list of folder names."""
 965         result = []
 966         for entry in os.listdir(self._path):
 967             if os.path.isdir(os.path.join(self._path, entry)):
 968                 result.append(entry)
 969         return result
 970
 971     def get_folder(self, folder):
 972         """Return an MH instance for the named folder."""
 973         return MH(os.path.join(self._path, folder),
 974                   factory=self._factory, create=False)
 975
 976     def add_folder(self, folder):
 977         """Create a folder and return an MH instance representing it."""
 978         return MH(os.path.join(self._path, folder),
 979                   factory=self._factory)
 980
 981     def remove_folder(self, folder):
 982         """Delete the named folder, which must be empty."""
 983         path = os.path.join(self._path, folder)
 984         entries = os.listdir(path)
 985         if entries == ['.mh_sequences']:
 986             os.remove(os.path.join(path, '.mh_sequences'))
 987         elif entries == []:
 988             pass
 989         else:
 990             raise NotEmptyError('Folder not empty: %s' % self._path)
 991         os.rmdir(path)
 992
 993     def get_sequences(self):
 994         """Return a name-to-key-list dictionary to define each sequence."""
 995         results = {}
 996         f = open(os.path.join(self._path, '.mh_sequences'), 'r')
 997         try:
 998             all_keys = set(self.keys())
 999             for line in f:
1000                 try:
1001                     name, contents = line.split(':')
1002                     keys = set()
1003                     for spec in contents.split():
1004                         if spec.isdigit():
1005                             keys.add(int(spec))
1006                         else:
1007                             start, stop = (int(x) for x in spec.split('-'))
1008                             keys.update(range(start, stop + 1))
1009                     results[name] = [key for key in sorted(keys) \
1010                                          if key in all_keys]
1011                     if len(results[name]) == 0:
1012                         del results[name]
1013                 except ValueError:
1014                     raise FormatError('Invalid sequence specification: %s' %
1015                                       line.rstrip())
1016         finally:
1017             f.close()
1018         return results
1019
1020     def set_sequences(self, sequences):
1021         """Set sequences using the given name-to-key-list dictionary."""
1022         f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
1023         try:
1024             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1025             for name, keys in sequences.iteritems():
1026                 if len(keys) == 0:
1027                     continue
1028                 f.write('%s:' % name)
1029                 prev = None
1030                 completing = False
1031                 for key in sorted(set(keys)):
1032                     if key - 1 == prev:
1033                         if not completing:
1034                             completing = True
1035                             f.write('-')
1036                     elif completing:
1037                         completing = False
1038                         f.write('%s %s' % (prev, key))
1039                     else:
1040                         f.write(' %s' % key)
1041                     prev = key
1042                 if completing:
1043                     f.write(str(prev) + '\n')
1044                 else:
1045                     f.write('\n')
1046         finally:
1047             _sync_close(f)
1048
1049     def pack(self):
1050         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1051         sequences = self.get_sequences()
1052         prev = 0
1053         changes = []
1054         for key in self.iterkeys():
1055             if key - 1 != prev:
1056                 changes.append((key, prev + 1))
1057                 if hasattr(os, 'link'):
1058                     os.link(os.path.join(self._path, str(key)),
1059                             os.path.join(self._path, str(prev + 1)))
1060                     os.unlink(os.path.join(self._path, str(key)))
1061                 else:
1062                     os.rename(os.path.join(self._path, str(key)),
1063                               os.path.join(self._path, str(prev + 1)))
1064             prev += 1
1065         self._next_key = prev + 1
1066         if len(changes) == 0:
1067             return
1068         for name, key_list in sequences.items():
1069             for old, new in changes:
1070                 if old in key_list:
1071                     key_list[key_list.index(old)] = new
1072         self.set_sequences(sequences)
1073
1074     def _dump_sequences(self, message, key):
1075         """Inspect a new MHMessage and update sequences appropriately."""
1076         pending_sequences = message.get_sequences()
1077         all_sequences = self.get_sequences()
1078         for name, key_list in all_sequences.iteritems():
1079             if name in pending_sequences:
1080                 key_list.append(key)
1081             elif key in key_list:
1082                 del key_list[key_list.index(key)]
1083         for sequence in pending_sequences:
1084             if sequence not in all_sequences:
1085                 all_sequences[sequence] = [key]
1086         self.set_sequences(all_sequences)
1087
1088
1089 class Babyl(_singlefileMailbox):
1090     """An Rmail-style Babyl mailbox."""
1091
1092     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1093                                  'forwarded', 'edited', 'resent'))
1094
1095     def __init__(self, path, factory=None, create=True):
1096         """Initialize a Babyl mailbox."""
1097         _singlefileMailbox.__init__(self, path, factory, create)
1098         self._labels = {}
1099
1100     def add(self, message):
1101         """Add message and return assigned key."""
1102         key = _singlefileMailbox.add(self, message)
1103         if isinstance(message, BabylMessage):
1104             self._labels[key] = message.get_labels()
1105         return key
1106
1107     def remove(self, key):
1108         """Remove the keyed message; raise KeyError if it doesn't exist."""
1109         _singlefileMailbox.remove(self, key)
1110         if key in self._labels:
1111             del self._labels[key]
1112
1113     def __setitem__(self, key, message):
1114         """Replace the keyed message; raise KeyError if it doesn't exist."""
1115         _singlefileMailbox.__setitem__(self, key, message)
1116         if isinstance(message, BabylMessage):
1117             self._labels[key] = message.get_labels()
1118
1119     def get_message(self, key):
1120         """Return a Message representation or raise a KeyError."""
1121         start, stop = self._lookup(key)
1122         self._file.seek(start)
1123         self._file.readline()   # Skip '1,' line specifying labels.
1124         original_headers = StringIO.StringIO()
1125         while True:
1126             line = self._file.readline()
1127             if line == '*** EOOH ***' + os.linesep or line == '':
1128                 break
1129             original_headers.write(line.replace(os.linesep, '\n'))
1130         visible_headers = StringIO.StringIO()
1131         while True:
1132             line = self._file.readline()
1133             if line == os.linesep or line == '':
1134                 break
1135             visible_headers.write(line.replace(os.linesep, '\n'))
1136         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1137                                                                  '\n')
1138         msg = BabylMessage(original_headers.getvalue() + body)
1139         msg.set_visible(visible_headers.getvalue())
1140         if key in self._labels:
1141             msg.set_labels(self._labels[key])
1142         return msg
1143
1144     def get_string(self, key):
1145         """Return a string representation or raise a KeyError."""
1146         start, stop = self._lookup(key)
1147         self._file.seek(start)
1148         self._file.readline()   # Skip '1,' line specifying labels.
1149         original_headers = StringIO.StringIO()
1150         while True:
1151             line = self._file.readline()
1152             if line == '*** EOOH ***' + os.linesep or line == '':
1153                 break
1154             original_headers.write(line.replace(os.linesep, '\n'))
1155         while True:
1156             line = self._file.readline()
1157             if line == os.linesep or line == '':
1158                 break
1159         return original_headers.getvalue() + \
1160                self._file.read(stop - self._file.tell()).replace(os.linesep,
1161                                                                  '\n')
1162
1163     def get_file(self, key):
1164         """Return a file-like representation or raise a KeyError."""
1165         return StringIO.StringIO(self.get_string(key).replace('\n',
1166                                                               os.linesep))
1167
1168     def get_labels(self):
1169         """Return a list of user-defined labels in the mailbox."""
1170         self._lookup()
1171         labels = set()
1172         for label_list in self._labels.values():
1173             labels.update(label_list)
1174         labels.difference_update(self._special_labels)
1175         return list(labels)
1176
1177     def _generate_toc(self):
1178         """Generate key-to-(start, stop) table of contents."""
1179         starts, stops = [], []
1180         self._file.seek(0)
1181         next_pos = 0
1182         label_lists = []
1183         while True:
1184             line_pos = next_pos
1185             line = self._file.readline()
1186             next_pos = self._file.tell()
1187             if line == '\037\014' + os.linesep:
1188                 if len(stops) < len(starts):
1189                     stops.append(line_pos - len(os.linesep))
1190                 starts.append(next_pos)
1191                 labels = [label.strip() for label
1192                                         in self._file.readline()[1:].split(',')
1193                                         if label.strip() != '']
1194                 label_lists.append(labels)
1195             elif line == '\037' or line == '\037' + os.linesep:
1196                 if len(stops) < len(starts):
1197                     stops.append(line_pos - len(os.linesep))
1198             elif line == '':
1199                 stops.append(line_pos - len(os.linesep))
1200                 break
1201         self._toc = dict(enumerate(zip(starts, stops)))
1202         self._labels = dict(enumerate(label_lists))
1203         self._next_key = len(self._toc)
1204
1205     def _pre_mailbox_hook(self, f):
1206         """Called before writing the mailbox to file f."""
1207         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1208                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1209                  os.linesep))
1210
1211     def _pre_message_hook(self, f):
1212         """Called before writing each message to file f."""
1213         f.write('\014' + os.linesep)
1214
1215     def _post_message_hook(self, f):
1216         """Called after writing each message to file f."""
1217         f.write(os.linesep + '\037')
1218
1219     def _install_message(self, message):
1220         """Write message contents and return (start, stop)."""
1221         start = self._file.tell()
1222         if isinstance(message, BabylMessage):
1223             special_labels = []
1224             labels = []
1225             for label in message.get_labels():
1226                 if label in self._special_labels:
1227                     special_labels.append(label)
1228                 else:
1229                     labels.append(label)
1230             self._file.write('1')
1231             for label in special_labels:
1232                 self._file.write(', ' + label)
1233             self._file.write(',,')
1234             for label in labels:
1235                 self._file.write(' ' + label + ',')
1236             self._file.write(os.linesep)
1237         else:
1238             self._file.write('1,,' + os.linesep)
1239         if isinstance(message, email.Message.Message):
1240             orig_buffer = StringIO.StringIO()
1241             orig_generator = email.Generator.Generator(orig_buffer, False, 0)
1242             orig_generator.flatten(message)
1243             orig_buffer.seek(0)
1244             while True:
1245                 line = orig_buffer.readline()
1246                 self._file.write(line.replace('\n', os.linesep))
1247                 if line == '\n' or line == '':
1248                     break
1249             self._file.write('*** EOOH ***' + os.linesep)
1250             if isinstance(message, BabylMessage):
1251                 vis_buffer = StringIO.StringIO()
1252                 vis_generator = email.Generator.Generator(vis_buffer, False, 0)
1253                 vis_generator.flatten(message.get_visible())
1254                 while True:
1255                     line = vis_buffer.readline()
1256                     self._file.write(line.replace('\n', os.linesep))
1257                     if line == '\n' or line == '':
1258                         break
1259             else:
1260                 orig_buffer.seek(0)
1261                 while True:
1262                     line = orig_buffer.readline()
1263                     self._file.write(line.replace('\n', os.linesep))
1264                     if line == '\n' or line == '':
1265                         break
1266             while True:
1267                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1268                 if buffer == '':
1269                     break
1270                 self._file.write(buffer.replace('\n', os.linesep))
1271         elif isinstance(message, str):
1272             body_start = message.find('\n\n') + 2
1273             if body_start - 2 != -1:
1274                 self._file.write(message[:body_start].replace('\n',
1275                                                               os.linesep))
1276                 self._file.write('*** EOOH ***' + os.linesep)
1277                 self._file.write(message[:body_start].replace('\n',
1278                                                               os.linesep))
1279                 self._file.write(message[body_start:].replace('\n',
1280                                                               os.linesep))
1281             else:
1282                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1283                 self._file.write(message.replace('\n', os.linesep))
1284         elif hasattr(message, 'readline'):
1285             original_pos = message.tell()
1286             first_pass = True
1287             while True:
1288                 line = message.readline()
1289                 self._file.write(line.replace('\n', os.linesep))
1290                 if line == '\n' or line == '':
1291                     self._file.write('*** EOOH ***' + os.linesep)
1292                     if first_pass:
1293                         first_pass = False
1294                         message.seek(original_pos)
1295                     else:
1296                         break
1297             while True:
1298                 buffer = message.read(4096)     # Buffer size is arbitrary.
1299                 if buffer == '':
1300                     break
1301                 self._file.write(buffer.replace('\n', os.linesep))
1302         else:
1303             raise TypeError('Invalid message type: %s' % type(message))
1304         stop = self._file.tell()
1305         return (start, stop)
1306
1307
1308 class Message(email.Message.Message):
1309     """Message with mailbox-format-specific properties."""
1310
1311     def __init__(self, message=None):
1312         """Initialize a Message instance."""
1313         if isinstance(message, email.Message.Message):
1314             self._become_message(copy.deepcopy(message))
1315             if isinstance(message, Message):
1316                 message._explain_to(self)
1317         elif isinstance(message, str):
1318             self._become_message(email.message_from_string(message))
1319         elif hasattr(message, "read"):
1320             self._become_message(email.message_from_file(message))
1321         elif message is None:
1322             email.Message.Message.__init__(self)
1323         else:
1324             raise TypeError('Invalid message type: %s' % type(message))
1325
1326     def _become_message(self, message):
1327         """Assume the non-format-specific state of message."""
1328         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1329                      'preamble', 'epilogue', 'defects', '_default_type'):
1330             self.__dict__[name] = message.__dict__[name]
1331
1332     def _explain_to(self, message):
1333         """Copy format-specific state to message insofar as possible."""
1334         if isinstance(message, Message):
1335             return  # There's nothing format-specific to explain.
1336         else:
1337             raise TypeError('Cannot convert to specified type')
1338
1339
1340 class MaildirMessage(Message):
1341     """Message with Maildir-specific properties."""
1342
1343     def __init__(self, message=None):
1344         """Initialize a MaildirMessage instance."""
1345         self._subdir = 'new'
1346         self._info = ''
1347         self._date = time.time()
1348         Message.__init__(self, message)
1349
1350     def get_subdir(self):
1351         """Return 'new' or 'cur'."""
1352         return self._subdir
1353
1354     def set_subdir(self, subdir):
1355         """Set subdir to 'new' or 'cur'."""
1356         if subdir == 'new' or subdir == 'cur':
1357             self._subdir = subdir
1358         else:
1359             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1360
1361     def get_flags(self):
1362         """Return as a string the flags that are set."""
1363         if self._info.startswith('2,'):
1364             return self._info[2:]
1365         else:
1366             return ''
1367
1368     def set_flags(self, flags):
1369         """Set the given flags and unset all others."""
1370         self._info = '2,' + ''.join(sorted(flags))
1371
1372     def add_flag(self, flag):
1373         """Set the given flag(s) without changing others."""
1374         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1375
1376     def remove_flag(self, flag):
1377         """Unset the given string flag(s) without changing others."""
1378         if self.get_flags() != '':
1379             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1380
1381     def get_date(self):
1382         """Return delivery date of message, in seconds since the epoch."""
1383         return self._date
1384
1385     def set_date(self, date):
1386         """Set delivery date of message, in seconds since the epoch."""
1387         try:
1388             self._date = float(date)
1389         except ValueError:
1390             raise TypeError("can't convert to float: %s" % date)
1391
1392     def get_info(self):
1393         """Get the message's "info" as a string."""
1394         return self._info
1395
1396     def set_info(self, info):
1397         """Set the message's "info" string."""
1398         if isinstance(info, str):
1399             self._info = info
1400         else:
1401             raise TypeError('info must be a string: %s' % type(info))
1402
1403     def _explain_to(self, message):
1404         """Copy Maildir-specific state to message insofar as possible."""
1405         if isinstance(message, MaildirMessage):
1406             message.set_flags(self.get_flags())
1407             message.set_subdir(self.get_subdir())
1408             message.set_date(self.get_date())
1409         elif isinstance(message, _mboxMMDFMessage):
1410             flags = set(self.get_flags())
1411             if 'S' in flags:
1412                 message.add_flag('R')
1413             if self.get_subdir() == 'cur':
1414                 message.add_flag('O')
1415             if 'T' in flags:
1416                 message.add_flag('D')
1417             if 'F' in flags:
1418                 message.add_flag('F')
1419             if 'R' in flags:
1420                 message.add_flag('A')
1421             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1422         elif isinstance(message, MHMessage):
1423             flags = set(self.get_flags())
1424             if 'S' not in flags:
1425                 message.add_sequence('unseen')
1426             if 'R' in flags:
1427                 message.add_sequence('replied')
1428             if 'F' in flags:
1429                 message.add_sequence('flagged')
1430         elif isinstance(message, BabylMessage):
1431             flags = set(self.get_flags())
1432             if 'S' not in flags:
1433                 message.add_label('unseen')
1434             if 'T' in flags:
1435                 message.add_label('deleted')
1436             if 'R' in flags:
1437                 message.add_label('answered')
1438             if 'P' in flags:
1439                 message.add_label('forwarded')
1440         elif isinstance(message, Message):
1441             pass
1442         else:
1443             raise TypeError('Cannot convert to specified type: %s' %
1444                             type(message))
1445
1446
1447 class _mboxMMDFMessage(Message):
1448     """Message with mbox- or MMDF-specific properties."""
1449
1450     def __init__(self, message=None):
1451         """Initialize an mboxMMDFMessage instance."""
1452         self.set_from('MAILER-DAEMON', True)
1453         if isinstance(message, email.Message.Message):
1454             unixfrom = message.get_unixfrom()
1455             if unixfrom is not None and unixfrom.startswith('From '):
1456                 self.set_from(unixfrom[5:])
1457         Message.__init__(self, message)
1458
1459     def get_from(self):
1460         """Return contents of "From " line."""
1461         return self._from
1462
1463     def set_from(self, from_, time_=None):
1464         """Set "From " line, formatting and appending time_ if specified."""
1465         if time_ is not None:
1466             if time_ is True:
1467                 time_ = time.gmtime()
1468             from_ += ' ' + time.asctime(time_)
1469         self._from = from_
1470
1471     def get_flags(self):
1472         """Return as a string the flags that are set."""
1473         return self.get('Status', '') + self.get('X-Status', '')
1474
1475     def set_flags(self, flags):
1476         """Set the given flags and unset all others."""
1477         flags = set(flags)
1478         status_flags, xstatus_flags = '', ''
1479         for flag in ('R', 'O'):
1480             if flag in flags:
1481                 status_flags += flag
1482                 flags.remove(flag)
1483         for flag in ('D', 'F', 'A'):
1484             if flag in flags:
1485                 xstatus_flags += flag
1486                 flags.remove(flag)
1487         xstatus_flags += ''.join(sorted(flags))
1488         try:
1489             self.replace_header('Status', status_flags)
1490         except KeyError:
1491             self.add_header('Status', status_flags)
1492         try:
1493             self.replace_header('X-Status', xstatus_flags)
1494         except KeyError:
1495             self.add_header('X-Status', xstatus_flags)
1496
1497     def add_flag(self, flag):
1498         """Set the given flag(s) without changing others."""
1499         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1500
1501     def remove_flag(self, flag):
1502         """Unset the given string flag(s) without changing others."""
1503         if 'Status' in self or 'X-Status' in self:
1504             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1505
1506     def _explain_to(self, message):
1507         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1508         if isinstance(message, MaildirMessage):
1509             flags = set(self.get_flags())
1510             if 'O' in flags:
1511                 message.set_subdir('cur')
1512             if 'F' in flags:
1513                 message.add_flag('F')
1514             if 'A' in flags:
1515                 message.add_flag('R')
1516             if 'R' in flags:
1517                 message.add_flag('S')
1518             if 'D' in flags:
1519                 message.add_flag('T')
1520             del message['status']
1521             del message['x-status']
1522             maybe_date = ' '.join(self.get_from().split()[-5:])
1523             try:
1524                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1525                                                       '%a %b %d %H:%M:%S %Y')))
1526             except (ValueError, OverflowError):
1527                 pass
1528         elif isinstance(message, _mboxMMDFMessage):
1529             message.set_flags(self.get_flags())
1530             message.set_from(self.get_from())
1531         elif isinstance(message, MHMessage):
1532             flags = set(self.get_flags())
1533             if 'R' not in flags:
1534                 message.add_sequence('unseen')
1535             if 'A' in flags:
1536                 message.add_sequence('replied')
1537             if 'F' in flags:
1538                 message.add_sequence('flagged')
1539             del message['status']
1540             del message['x-status']
1541         elif isinstance(message, BabylMessage):
1542             flags = set(self.get_flags())
1543             if 'R' not in flags:
1544                 message.add_label('unseen')
1545             if 'D' in flags:
1546                 message.add_label('deleted')
1547             if 'A' in flags:
1548                 message.add_label('answered')
1549             del message['status']
1550             del message['x-status']
1551         elif isinstance(message, Message):
1552             pass
1553         else:
1554             raise TypeError('Cannot convert to specified type: %s' %
1555                             type(message))
1556
1557
1558 class mboxMessage(_mboxMMDFMessage):
1559     """Message with mbox-specific properties."""
1560
1561
1562 class MHMessage(Message):
1563     """Message with MH-specific properties."""
1564
1565     def __init__(self, message=None):
1566         """Initialize an MHMessage instance."""
1567         self._sequences = []
1568         Message.__init__(self, message)
1569
1570     def get_sequences(self):
1571         """Return a list of sequences that include the message."""
1572         return self._sequences[:]
1573
1574     def set_sequences(self, sequences):
1575         """Set the list of sequences that include the message."""
1576         self._sequences = list(sequences)
1577
1578     def add_sequence(self, sequence):
1579         """Add sequence to list of sequences including the message."""
1580         if isinstance(sequence, str):
1581             if not sequence in self._sequences:
1582                 self._sequences.append(sequence)
1583         else:
1584             raise TypeError('sequence must be a string: %s' % type(sequence))
1585
1586     def remove_sequence(self, sequence):
1587         """Remove sequence from the list of sequences including the message."""
1588         try:
1589             self._sequences.remove(sequence)
1590         except ValueError:
1591             pass
1592
1593     def _explain_to(self, message):
1594         """Copy MH-specific state to message insofar as possible."""
1595         if isinstance(message, MaildirMessage):
1596             sequences = set(self.get_sequences())
1597             if 'unseen' in sequences:
1598                 message.set_subdir('cur')
1599             else:
1600                 message.set_subdir('cur')
1601                 message.add_flag('S')
1602             if 'flagged' in sequences:
1603                 message.add_flag('F')
1604             if 'replied' in sequences:
1605                 message.add_flag('R')
1606         elif isinstance(message, _mboxMMDFMessage):
1607             sequences = set(self.get_sequences())
1608             if 'unseen' not in sequences:
1609                 message.add_flag('RO')
1610             else:
1611                 message.add_flag('O')
1612             if 'flagged' in sequences:
1613                 message.add_flag('F')
1614             if 'replied' in sequences:
1615                 message.add_flag('A')
1616         elif isinstance(message, MHMessage):
1617             for sequence in self.get_sequences():
1618                 message.add_sequence(sequence)
1619         elif isinstance(message, BabylMessage):
1620             sequences = set(self.get_sequences())
1621             if 'unseen' in sequences:
1622                 message.add_label('unseen')
1623             if 'replied' in sequences:
1624                 message.add_label('answered')
1625         elif isinstance(message, Message):
1626             pass
1627         else:
1628             raise TypeError('Cannot convert to specified type: %s' %
1629                             type(message))
1630
1631
1632 class BabylMessage(Message):
1633     """Message with Babyl-specific properties."""
1634
1635     def __init__(self, message=None):
1636         """Initialize an BabylMessage instance."""
1637         self._labels = []
1638         self._visible = Message()
1639         Message.__init__(self, message)
1640
1641     def get_labels(self):
1642         """Return a list of labels on the message."""
1643         return self._labels[:]
1644
1645     def set_labels(self, labels):
1646         """Set the list of labels on the message."""
1647         self._labels = list(labels)
1648
1649     def add_label(self, label):
1650         """Add label to list of labels on the message."""
1651         if isinstance(label, str):
1652             if label not in self._labels:
1653                 self._labels.append(label)
1654         else:
1655             raise TypeError('label must be a string: %s' % type(label))
1656
1657     def remove_label(self, label):
1658         """Remove label from the list of labels on the message."""
1659         try:
1660             self._labels.remove(label)
1661         except ValueError:
1662             pass
1663
1664     def get_visible(self):
1665         """Return a Message representation of visible headers."""
1666         return Message(self._visible)
1667
1668     def set_visible(self, visible):
1669         """Set the Message representation of visible headers."""
1670         self._visible = Message(visible)
1671
1672     def update_visible(self):
1673         """Update and/or sensibly generate a set of visible headers."""
1674         for header in self._visible.keys():
1675             if header in self:
1676                 self._visible.replace_header(header, self[header])
1677             else:
1678                 del self._visible[header]
1679         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1680             if header in self and header not in self._visible:
1681                 self._visible[header] = self[header]
1682
1683     def _explain_to(self, message):
1684         """Copy Babyl-specific state to message insofar as possible."""
1685         if isinstance(message, MaildirMessage):
1686             labels = set(self.get_labels())
1687             if 'unseen' in labels:
1688                 message.set_subdir('cur')
1689             else:
1690                 message.set_subdir('cur')
1691                 message.add_flag('S')
1692             if 'forwarded' in labels or 'resent' in labels:
1693                 message.add_flag('P')
1694             if 'answered' in labels:
1695                 message.add_flag('R')
1696             if 'deleted' in labels:
1697                 message.add_flag('T')
1698         elif isinstance(message, _mboxMMDFMessage):
1699             labels = set(self.get_labels())
1700             if 'unseen' not in labels:
1701                 message.add_flag('RO')
1702             else:
1703                 message.add_flag('O')
1704             if 'deleted' in labels:
1705                 message.add_flag('D')
1706             if 'answered' in labels:
1707                 message.add_flag('A')
1708         elif isinstance(message, MHMessage):
1709             labels = set(self.get_labels())
1710             if 'unseen' in labels:
1711                 message.add_sequence('unseen')
1712             if 'answered' in labels:
1713                 message.add_sequence('replied')
1714         elif isinstance(message, BabylMessage):
1715             message.set_visible(self.get_visible())
1716             for label in self.get_labels():
1717                 message.add_label(label)
1718         elif isinstance(message, Message):
1719             pass
1720         else:
1721             raise TypeError('Cannot convert to specified type: %s' %
1722                             type(message))
1723
1724
1725 class MMDFMessage(_mboxMMDFMessage):
1726     """Message with MMDF-specific properties."""
1727
1728
1729 class _ProxyFile:
1730     """A read-only wrapper of a file."""
1731
1732     def __init__(self, f, pos=None):
1733         """Initialize a _ProxyFile."""
1734         self._file = f
1735         if pos is None:
1736             self._pos = f.tell()
1737         else:
1738             self._pos = pos
1739
1740     def read(self, size=None):
1741         """Read bytes."""
1742         return self._read(size, self._file.read)
1743
1744     def readline(self, size=None):
1745         """Read a line."""
1746         return self._read(size, self._file.readline)
1747
1748     def readlines(self, sizehint=None):
1749         """Read multiple lines."""
1750         result = []
1751         for line in self:
1752             result.append(line)
1753             if sizehint is not None:
1754                 sizehint -= len(line)
1755                 if sizehint <= 0:
1756                     break
1757         return result
1758
1759     def __iter__(self):
1760         """Iterate over lines."""
1761         return iter(self.readline, "")
1762
1763     def tell(self):
1764         """Return the position."""
1765         return self._pos
1766
1767     def seek(self, offset, whence=0):
1768         """Change position."""
1769         if whence == 1:
1770             self._file.seek(self._pos)
1771         self._file.seek(offset, whence)
1772         self._pos = self._file.tell()
1773
1774     def close(self):
1775         """Close the file."""
1776         del self._file
1777
1778     def _read(self, size, read_method):
1779         """Read size bytes using read_method."""
1780         if size is None:
1781             size = -1
1782         self._file.seek(self._pos)
1783         result = read_method(size)
1784         self._pos = self._file.tell()
1785         return result
1786
1787
1788 class _PartialFile(_ProxyFile):
1789     """A read-only wrapper of part of a file."""
1790
1791     def __init__(self, f, start=None, stop=None):
1792         """Initialize a _PartialFile."""
1793         _ProxyFile.__init__(self, f, start)
1794         self._start = start
1795         self._stop = stop
1796
1797     def tell(self):
1798         """Return the position with respect to start."""
1799         return _ProxyFile.tell(self) - self._start
1800
1801     def seek(self, offset, whence=0):
1802         """Change position, possibly with respect to start or stop."""
1803         if whence == 0:
1804             self._pos = self._start
1805             whence = 1
1806         elif whence == 2:
1807             self._pos = self._stop
1808             whence = 1
1809         _ProxyFile.seek(self, offset, whence)
1810
1811     def _read(self, size, read_method):
1812         """Read size bytes using read_method, honoring start and stop."""
1813         remaining = self._stop - self._pos
1814         if remaining <= 0:
1815             return ''
1816         if size is None or size < 0 or size > remaining:
1817             size = remaining
1818         return _ProxyFile._read(self, size, read_method)
1819
1820
1821 def _lock_file(f, dotlock=True):
1822     """Lock file f using lockf and dot locking."""
1823     dotlock_done = False
1824     try:
1825         if fcntl:
1826             try:
1827                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1828             except IOError, e:
1829                 if e.errno in (errno.EAGAIN, errno.EACCES):
1830                     raise ExternalClashError('lockf: lock unavailable: %s' %
1831                                              f.name)
1832                 else:
1833                     raise
1834         if dotlock:
1835             try:
1836                 pre_lock = _create_temporary(f.name + '.lock')
1837                 pre_lock.close()
1838             except IOError, e:
1839                 if e.errno == errno.EACCES:
1840                     return  # Without write access, just skip dotlocking.
1841                 else:
1842                     raise
1843             try:
1844                 if hasattr(os, 'link'):
1845                     os.link(pre_lock.name, f.name + '.lock')
1846                     dotlock_done = True
1847                     os.unlink(pre_lock.name)
1848                 else:
1849                     os.rename(pre_lock.name, f.name + '.lock')
1850                     dotlock_done = True
1851             except OSError, e:
1852                 if e.errno == errno.EEXIST or \
1853                   (os.name == 'os2' and e.errno == errno.EACCES):
1854                     os.remove(pre_lock.name)
1855                     raise ExternalClashError('dot lock unavailable: %s' %
1856                                              f.name)
1857                 else:
1858                     raise
1859     except:
1860         if fcntl:
1861             fcntl.lockf(f, fcntl.LOCK_UN)
1862         if dotlock_done:
1863             os.remove(f.name + '.lock')
1864         raise
1865
1866 def _unlock_file(f):
1867     """Unlock file f using lockf and dot locking."""
1868     if fcntl:
1869         fcntl.lockf(f, fcntl.LOCK_UN)
1870     if os.path.exists(f.name + '.lock'):
1871         os.remove(f.name + '.lock')
1872
1873 def _create_carefully(path):
1874     """Create a file if it doesn't exist and open for reading and writing."""
1875     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
1876     try:
1877         return open(path, 'rb+')
1878     finally:
1879         os.close(fd)
1880
1881 def _create_temporary(path):
1882     """Create a temp file based on path and open for reading and writing."""
1883     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1884                                               socket.gethostname(),
1885                                               os.getpid()))
1886
1887 def _sync_flush(f):
1888     """Ensure changes to file f are physically on disk."""
1889     f.flush()
1890     os.fsync(f.fileno())
1891
1892 def _sync_close(f):
1893     """Close file f, ensuring all changes are physically on disk."""
1894     _sync_flush(f)
1895     f.close()
1896
1897 ## Start: classes from the original module (for backward compatibility).
1898
1899 # Note that the Maildir class, whose name is unchanged, itself offers a next()
1900 # method for backward compatibility.
1901
1902 class _Mailbox:
1903
1904     def __init__(self, fp, factory=rfc822.Message):
1905         self.fp = fp
1906         self.seekp = 0
1907         self.factory = factory
1908
1909     def __iter__(self):
1910         return iter(self.next, None)
1911
1912     def next(self):
1913         while 1:
1914             self.fp.seek(self.seekp)
1915             try:
1916                 self._search_start()
1917             except EOFError:
1918                 self.seekp = self.fp.tell()
1919                 return None
1920             start = self.fp.tell()
1921             self._search_end()
1922             self.seekp = stop = self.fp.tell()
1923             if start != stop:
1924                 break
1925         return self.factory(_PartialFile(self.fp, start, stop))
1926
1927 # Recommended to use PortableUnixMailbox instead!
1928 class UnixMailbox(_Mailbox):
1929
1930     def _search_start(self):
1931         while 1:
1932             pos = self.fp.tell()
1933             line = self.fp.readline()
1934             if not line:
1935                 raise EOFError
1936             if line[:5] == 'From ' and self._isrealfromline(line):
1937                 self.fp.seek(pos)
1938                 return
1939
1940     def _search_end(self):
1941         self.fp.readline()      # Throw away header line
1942         while 1:
1943             pos = self.fp.tell()
1944             line = self.fp.readline()
1945             if not line:
1946                 return
1947             if line[:5] == 'From ' and self._isrealfromline(line):
1948                 self.fp.seek(pos)
1949                 return
1950
1951     # An overridable mechanism to test for From-line-ness.  You can either
1952     # specify a different regular expression or define a whole new
1953     # _isrealfromline() method.  Note that this only gets called for lines
1954     # starting with the 5 characters "From ".
1955     #
1956     # BAW: According to
1957     #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
1958     # the only portable, reliable way to find message delimiters in a BSD (i.e
1959     # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
1960     # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
1961     # like a good idea, in practice, there are too many variations for more
1962     # strict parsing of the line to be completely accurate.
1963     #
1964     # _strict_isrealfromline() is the old version which tries to do stricter
1965     # parsing of the From_ line.  _portable_isrealfromline() simply returns
1966     # true, since it's never called if the line doesn't already start with
1967     # "From ".
1968     #
1969     # This algorithm, and the way it interacts with _search_start() and
1970     # _search_end() may not be completely correct, because it doesn't check
1971     # that the two characters preceding "From " are \n\n or the beginning of
1972     # the file.  Fixing this would require a more extensive rewrite than is
1973     # necessary.  For convenience, we've added a PortableUnixMailbox class
1974     # which uses the more lenient _fromlinepattern regular expression.
1975
1976     _fromlinepattern = r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+" \
1977                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*$"
1978     _regexp = None
1979
1980     def _strict_isrealfromline(self, line):
1981         if not self._regexp:
1982             import re
1983             self._regexp = re.compile(self._fromlinepattern)
1984         return self._regexp.match(line)
1985
1986     def _portable_isrealfromline(self, line):
1987         return True
1988
1989     _isrealfromline = _strict_isrealfromline
1990
1991
1992 class PortableUnixMailbox(UnixMailbox):
1993     _isrealfromline = UnixMailbox._portable_isrealfromline
1994
1995
1996 class MmdfMailbox(_Mailbox):
1997
1998     def _search_start(self):
1999         while 1:
2000             line = self.fp.readline()
2001             if not line:
2002                 raise EOFError
2003             if line[:5] == '\001\001\001\001\n':
2004                 return
2005
2006     def _search_end(self):
2007         while 1:
2008             pos = self.fp.tell()
2009             line = self.fp.readline()
2010             if not line:
2011                 return
2012             if line == '\001\001\001\001\n':
2013                 self.fp.seek(pos)
2014                 return
2015
2016
2017 class MHMailbox:
2018
2019     def __init__(self, dirname, factory=rfc822.Message):
2020         import re
2021         pat = re.compile('^[1-9][0-9]*$')
2022         self.dirname = dirname
2023         # the three following lines could be combined into:
2024         # list = map(long, filter(pat.match, os.listdir(self.dirname)))
2025         list = os.listdir(self.dirname)
2026         list = filter(pat.match, list)
2027         list = map(long, list)
2028         list.sort()
2029         # This only works in Python 1.6 or later;
2030         # before that str() added 'L':
2031         self.boxes = map(str, list)
2032         self.boxes.reverse()
2033         self.factory = factory
2034
2035     def __iter__(self):
2036         return iter(self.next, None)
2037
2038     def next(self):
2039         if not self.boxes:
2040             return None
2041         fn = self.boxes.pop()
2042         fp = open(os.path.join(self.dirname, fn))
2043         msg = self.factory(fp)
2044         try:
2045             msg._mh_msgno = fn
2046         except (AttributeError, TypeError):
2047             pass
2048         return msg
2049
2050
2051 class BabylMailbox(_Mailbox):
2052
2053     def _search_start(self):
2054         while 1:
2055             line = self.fp.readline()
2056             if not line:
2057                 raise EOFError
2058             if line == '*** EOOH ***\n':
2059                 return
2060
2061     def _search_end(self):
2062         while 1:
2063             pos = self.fp.tell()
2064             line = self.fp.readline()
2065             if not line:
2066                 return
2067             if line == '\037\014\n' or line == '\037':
2068                 self.fp.seek(pos)
2069                 return
2070
2071 ## End: classes from the original module (for backward compatibility).
2072
2073
2074 class Error(Exception):
2075     """Raised for module-specific errors."""
2076
2077 class NoSuchMailboxError(Error):
2078     """The specified mailbox does not exist and won't be created."""
2079
2080 class NotEmptyError(Error):
2081     """The specified mailbox is not empty and deletion was requested."""
2082
2083 class ExternalClashError(Error):
2084     """Another process caused an action to fail."""
2085
2086 class FormatError(Error):
2087     """A file appears to have an invalid format."""