Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.message
  20 import email.generator
  21 import rfc822
  22 import StringIO
  23 try:
  24     if sys.platform == 'os2emx':
  25         # OS/2 EMX fcntl() not adequate
  26         raise ImportError
  27     import fcntl
  28 except ImportError:
  29     fcntl = None
  30
  31 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  32             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  33             'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  34             'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  35
  36 class Mailbox:
  37     """A group of messages in a particular place."""
  38
  39     def __init__(self, path, factory=None, create=True):
  40         """Initialize a Mailbox instance."""
  41         self._path = os.path.abspath(os.path.expanduser(path))
  42         self._factory = factory
  43
  44     def add(self, message):
  45         """Add message and return assigned key."""
  46         raise NotImplementedError('Method must be implemented by subclass')
  47
  48     def remove(self, key):
  49         """Remove the keyed message; raise KeyError if it doesn't exist."""
  50         raise NotImplementedError('Method must be implemented by subclass')
  51
  52     def __delitem__(self, key):
  53         self.remove(key)
  54
  55     def discard(self, key):
  56         """If the keyed message exists, remove it."""
  57         try:
  58             self.remove(key)
  59         except KeyError:
  60             pass
  61
  62     def __setitem__(self, key, message):
  63         """Replace the keyed message; raise KeyError if it doesn't exist."""
  64         raise NotImplementedError('Method must be implemented by subclass')
  65
  66     def get(self, key, default=None):
  67         """Return the keyed message, or default if it doesn't exist."""
  68         try:
  69             return self.__getitem__(key)
  70         except KeyError:
  71             return default
  72
  73     def __getitem__(self, key):
  74         """Return the keyed message; raise KeyError if it doesn't exist."""
  75         if not self._factory:
  76             return self.get_message(key)
  77         else:
  78             return self._factory(self.get_file(key))
  79
  80     def get_message(self, key):
  81         """Return a Message representation or raise a KeyError."""
  82         raise NotImplementedError('Method must be implemented by subclass')
  83
  84     def get_string(self, key):
  85         """Return a string representation or raise a KeyError."""
  86         raise NotImplementedError('Method must be implemented by subclass')
  87
  88     def get_file(self, key):
  89         """Return a file-like representation or raise a KeyError."""
  90         raise NotImplementedError('Method must be implemented by subclass')
  91
  92     def iterkeys(self):
  93         """Return an iterator over keys."""
  94         raise NotImplementedError('Method must be implemented by subclass')
  95
  96     def keys(self):
  97         """Return a list of keys."""
  98         return list(self.iterkeys())
  99
 100     def itervalues(self):
 101         """Return an iterator over all messages."""
 102         for key in self.iterkeys():
 103             try:
 104                 value = self[key]
 105             except KeyError:
 106                 continue
 107             yield value
 108
 109     def __iter__(self):
 110         return self.itervalues()
 111
 112     def values(self):
 113         """Return a list of messages. Memory intensive."""
 114         return list(self.itervalues())
 115
 116     def iteritems(self):
 117         """Return an iterator over (key, message) tuples."""
 118         for key in self.iterkeys():
 119             try:
 120                 value = self[key]
 121             except KeyError:
 122                 continue
 123             yield (key, value)
 124
 125     def items(self):
 126         """Return a list of (key, message) tuples. Memory intensive."""
 127         return list(self.iteritems())
 128
 129     def has_key(self, key):
 130         """Return True if the keyed message exists, False otherwise."""
 131         raise NotImplementedError('Method must be implemented by subclass')
 132
 133     def __contains__(self, key):
 134         return self.has_key(key)
 135
 136     def __len__(self):
 137         """Return a count of messages in the mailbox."""
 138         raise NotImplementedError('Method must be implemented by subclass')
 139
 140     def clear(self):
 141         """Delete all messages."""
 142         for key in self.iterkeys():
 143             self.discard(key)
 144
 145     def pop(self, key, default=None):
 146         """Delete the keyed message and return it, or default."""
 147         try:
 148             result = self[key]
 149         except KeyError:
 150             return default
 151         self.discard(key)
 152         return result
 153
 154     def popitem(self):
 155         """Delete an arbitrary (key, message) pair and return it."""
 156         for key in self.iterkeys():
 157             return (key, self.pop(key))     # This is only run once.
 158         else:
 159             raise KeyError('No messages in mailbox')
 160
 161     def update(self, arg=None):
 162         """Change the messages that correspond to certain keys."""
 163         if hasattr(arg, 'iteritems'):
 164             source = arg.iteritems()
 165         elif hasattr(arg, 'items'):
 166             source = arg.items()
 167         else:
 168             source = arg
 169         bad_key = False
 170         for key, message in source:
 171             try:
 172                 self[key] = message
 173             except KeyError:
 174                 bad_key = True
 175         if bad_key:
 176             raise KeyError('No message with key(s)')
 177
 178     def flush(self):
 179         """Write any pending changes to the disk."""
 180         raise NotImplementedError('Method must be implemented by subclass')
 181
 182     def lock(self):
 183         """Lock the mailbox."""
 184         raise NotImplementedError('Method must be implemented by subclass')
 185
 186     def unlock(self):
 187         """Unlock the mailbox if it is locked."""
 188         raise NotImplementedError('Method must be implemented by subclass')
 189
 190     def close(self):
 191         """Flush and close the mailbox."""
 192         raise NotImplementedError('Method must be implemented by subclass')
 193
 194     def _dump_message(self, message, target, mangle_from_=False):
 195         # Most files are opened in binary mode to allow predictable seeking.
 196         # To get native line endings on disk, the user-friendly \n line endings
 197         # used in strings and by email.Message are translated here.
 198         """Dump message contents to target file."""
 199         if isinstance(message, email.message.Message):
 200             buffer = StringIO.StringIO()
 201             gen = email.generator.Generator(buffer, mangle_from_, 0)
 202             gen.flatten(message)
 203             buffer.seek(0)
 204             target.write(buffer.read().replace('\n', os.linesep))
 205         elif isinstance(message, str):
 206             if mangle_from_:
 207                 message = message.replace('\nFrom ', '\n>From ')
 208             message = message.replace('\n', os.linesep)
 209             target.write(message)
 210         elif hasattr(message, 'read'):
 211             while True:
 212                 line = message.readline()
 213                 if line == '':
 214                     break
 215                 if mangle_from_ and line.startswith('From '):
 216                     line = '>From ' + line[5:]
 217                 line = line.replace('\n', os.linesep)
 218                 target.write(line)
 219         else:
 220             raise TypeError('Invalid message type: %s' % type(message))
 221
 222
 223 class Maildir(Mailbox):
 224     """A qmail-style Maildir mailbox."""
 225
 226     colon = ':'
 227
 228     def __init__(self, dirname, factory=rfc822.Message, create=True):
 229         """Initialize a Maildir instance."""
 230         Mailbox.__init__(self, dirname, factory, create)
 231         if not os.path.exists(self._path):
 232             if create:
 233                 os.mkdir(self._path, 0700)
 234                 os.mkdir(os.path.join(self._path, 'tmp'), 0700)
 235                 os.mkdir(os.path.join(self._path, 'new'), 0700)
 236                 os.mkdir(os.path.join(self._path, 'cur'), 0700)
 237             else:
 238                 raise NoSuchMailboxError(self._path)
 239         self._toc = {}
 240         self._last_read = None          # Records last time we read cur/new
 241         # NOTE: we manually invalidate _last_read each time we do any
 242         # modifications ourselves, otherwise we might get tripped up by
 243         # bogus mtime behaviour on some systems (see issue #6896).
 244
 245     def add(self, message):
 246         """Add message and return assigned key."""
 247         tmp_file = self._create_tmp()
 248         try:
 249             self._dump_message(message, tmp_file)
 250         finally:
 251             _sync_close(tmp_file)
 252         if isinstance(message, MaildirMessage):
 253             subdir = message.get_subdir()
 254             suffix = self.colon + message.get_info()
 255             if suffix == self.colon:
 256                 suffix = ''
 257         else:
 258             subdir = 'new'
 259             suffix = ''
 260         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 261         dest = os.path.join(self._path, subdir, uniq + suffix)
 262         try:
 263             if hasattr(os, 'link'):
 264                 os.link(tmp_file.name, dest)
 265                 os.remove(tmp_file.name)
 266             else:
 267                 os.rename(tmp_file.name, dest)
 268         except OSError, e:
 269             os.remove(tmp_file.name)
 270             if e.errno == errno.EEXIST:
 271                 raise ExternalClashError('Name clash with existing message: %s'
 272                                          % dest)
 273             else:
 274                 raise
 275         if isinstance(message, MaildirMessage):
 276             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 277         # Invalidate cached toc
 278         self._last_read = None
 279         return uniq
 280
 281     def remove(self, key):
 282         """Remove the keyed message; raise KeyError if it doesn't exist."""
 283         os.remove(os.path.join(self._path, self._lookup(key)))
 284         # Invalidate cached toc (only on success)
 285         self._last_read = None
 286
 287     def discard(self, key):
 288         """If the keyed message exists, remove it."""
 289         # This overrides an inapplicable implementation in the superclass.
 290         try:
 291             self.remove(key)
 292         except KeyError:
 293             pass
 294         except OSError, e:
 295             if e.errno != errno.ENOENT:
 296                 raise
 297
 298     def __setitem__(self, key, message):
 299         """Replace the keyed message; raise KeyError if it doesn't exist."""
 300         old_subpath = self._lookup(key)
 301         temp_key = self.add(message)
 302         temp_subpath = self._lookup(temp_key)
 303         if isinstance(message, MaildirMessage):
 304             # temp's subdir and suffix were specified by message.
 305             dominant_subpath = temp_subpath
 306         else:
 307             # temp's subdir and suffix were defaults from add().
 308             dominant_subpath = old_subpath
 309         subdir = os.path.dirname(dominant_subpath)
 310         if self.colon in dominant_subpath:
 311             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 312         else:
 313             suffix = ''
 314         self.discard(key)
 315         new_path = os.path.join(self._path, subdir, key + suffix)
 316         os.rename(os.path.join(self._path, temp_subpath), new_path)
 317         if isinstance(message, MaildirMessage):
 318             os.utime(new_path, (os.path.getatime(new_path),
 319                                 message.get_date()))
 320         # Invalidate cached toc
 321         self._last_read = None
 322
 323     def get_message(self, key):
 324         """Return a Message representation or raise a KeyError."""
 325         subpath = self._lookup(key)
 326         f = open(os.path.join(self._path, subpath), 'r')
 327         try:
 328             if self._factory:
 329                 msg = self._factory(f)
 330             else:
 331                 msg = MaildirMessage(f)
 332         finally:
 333             f.close()
 334         subdir, name = os.path.split(subpath)
 335         msg.set_subdir(subdir)
 336         if self.colon in name:
 337             msg.set_info(name.split(self.colon)[-1])
 338         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 339         return msg
 340
 341     def get_string(self, key):
 342         """Return a string representation or raise a KeyError."""
 343         f = open(os.path.join(self._path, self._lookup(key)), 'r')
 344         try:
 345             return f.read()
 346         finally:
 347             f.close()
 348
 349     def get_file(self, key):
 350         """Return a file-like representation or raise a KeyError."""
 351         f = open(os.path.join(self._path, self._lookup(key)), 'rb')
 352         return _ProxyFile(f)
 353
 354     def iterkeys(self):
 355         """Return an iterator over keys."""
 356         self._refresh()
 357         for key in self._toc:
 358             try:
 359                 self._lookup(key)
 360             except KeyError:
 361                 continue
 362             yield key
 363
 364     def has_key(self, key):
 365         """Return True if the keyed message exists, False otherwise."""
 366         self._refresh()
 367         return key in self._toc
 368
 369     def __len__(self):
 370         """Return a count of messages in the mailbox."""
 371         self._refresh()
 372         return len(self._toc)
 373
 374     def flush(self):
 375         """Write any pending changes to disk."""
 376         # Maildir changes are always written immediately, so there's nothing
 377         # to do except invalidate our cached toc.
 378         self._last_read = None
 379
 380     def lock(self):
 381         """Lock the mailbox."""
 382         return
 383
 384     def unlock(self):
 385         """Unlock the mailbox if it is locked."""
 386         return
 387
 388     def close(self):
 389         """Flush and close the mailbox."""
 390         return
 391
 392     def list_folders(self):
 393         """Return a list of folder names."""
 394         result = []
 395         for entry in os.listdir(self._path):
 396             if len(entry) > 1 and entry[0] == '.' and \
 397                os.path.isdir(os.path.join(self._path, entry)):
 398                 result.append(entry[1:])
 399         return result
 400
 401     def get_folder(self, folder):
 402         """Return a Maildir instance for the named folder."""
 403         return Maildir(os.path.join(self._path, '.' + folder),
 404                        factory=self._factory,
 405                        create=False)
 406
 407     def add_folder(self, folder):
 408         """Create a folder and return a Maildir instance representing it."""
 409         path = os.path.join(self._path, '.' + folder)
 410         result = Maildir(path, factory=self._factory)
 411         maildirfolder_path = os.path.join(path, 'maildirfolder')
 412         if not os.path.exists(maildirfolder_path):
 413             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
 414                 0666))
 415         return result
 416
 417     def remove_folder(self, folder):
 418         """Delete the named folder, which must be empty."""
 419         path = os.path.join(self._path, '.' + folder)
 420         for entry in os.listdir(os.path.join(path, 'new')) + \
 421                      os.listdir(os.path.join(path, 'cur')):
 422             if len(entry) < 1 or entry[0] != '.':
 423                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 424         for entry in os.listdir(path):
 425             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 426                os.path.isdir(os.path.join(path, entry)):
 427                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 428                                     (folder, entry))
 429         for root, dirs, files in os.walk(path, topdown=False):
 430             for entry in files:
 431                 os.remove(os.path.join(root, entry))
 432             for entry in dirs:
 433                 os.rmdir(os.path.join(root, entry))
 434         os.rmdir(path)
 435
 436     def clean(self):
 437         """Delete old files in "tmp"."""
 438         now = time.time()
 439         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 440             path = os.path.join(self._path, 'tmp', entry)
 441             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 442                 os.remove(path)
 443
 444     _count = 1  # This is used to generate unique file names.
 445
 446     def _create_tmp(self):
 447         """Create a file in the tmp subdirectory and open and return it."""
 448         now = time.time()
 449         hostname = socket.gethostname()
 450         if '/' in hostname:
 451             hostname = hostname.replace('/', r'\057')
 452         if ':' in hostname:
 453             hostname = hostname.replace(':', r'\072')
 454         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 455                                     Maildir._count, hostname)
 456         path = os.path.join(self._path, 'tmp', uniq)
 457         try:
 458             os.stat(path)
 459         except OSError, e:
 460             if e.errno == errno.ENOENT:
 461                 Maildir._count += 1
 462                 try:
 463                     return _create_carefully(path)
 464                 except OSError, e:
 465                     if e.errno != errno.EEXIST:
 466                         raise
 467             else:
 468                 raise
 469
 470         # Fall through to here if stat succeeded or open raised EEXIST.
 471         raise ExternalClashError('Name clash prevented file creation: %s' %
 472                                  path)
 473
 474     def _refresh(self):
 475         """Update table of contents mapping."""
 476         if self._last_read is not None:
 477             for subdir in ('new', 'cur'):
 478                 mtime = os.path.getmtime(os.path.join(self._path, subdir))
 479                 if mtime > self._last_read:
 480                     break
 481             else:
 482                 return
 483
 484         # We record the current time - 1sec so that, if _refresh() is called
 485         # again in the same second, we will always re-read the mailbox
 486         # just in case it's been modified.  (os.path.mtime() only has
 487         # 1sec resolution.)  This results in a few unnecessary re-reads
 488         # when _refresh() is called multiple times in the same second,
 489         # but once the clock ticks over, we will only re-read as needed.
 490         now = time.time() - 1
 491
 492         self._toc = {}
 493         def update_dir (subdir):
 494             path = os.path.join(self._path, subdir)
 495             for entry in os.listdir(path):
 496                 p = os.path.join(path, entry)
 497                 if os.path.isdir(p):
 498                     continue
 499                 uniq = entry.split(self.colon)[0]
 500                 self._toc[uniq] = os.path.join(subdir, entry)
 501
 502         update_dir('new')
 503         update_dir('cur')
 504
 505         self._last_read = now
 506
 507     def _lookup(self, key):
 508         """Use TOC to return subpath for given key, or raise a KeyError."""
 509         try:
 510             if os.path.exists(os.path.join(self._path, self._toc[key])):
 511                 return self._toc[key]
 512         except KeyError:
 513             pass
 514         self._refresh()
 515         try:
 516             return self._toc[key]
 517         except KeyError:
 518             raise KeyError('No message with key: %s' % key)
 519
 520     # This method is for backward compatibility only.
 521     def next(self):
 522         """Return the next message in a one-time iteration."""
 523         if not hasattr(self, '_onetime_keys'):
 524             self._onetime_keys = self.iterkeys()
 525         while True:
 526             try:
 527                 return self[self._onetime_keys.next()]
 528             except StopIteration:
 529                 return None
 530             except KeyError:
 531                 continue
 532
 533
 534 class _singlefileMailbox(Mailbox):
 535     """A single-file mailbox."""
 536
 537     def __init__(self, path, factory=None, create=True):
 538         """Initialize a single-file mailbox."""
 539         Mailbox.__init__(self, path, factory, create)
 540         try:
 541             f = open(self._path, 'rb+')
 542         except IOError, e:
 543             if e.errno == errno.ENOENT:
 544                 if create:
 545                     f = open(self._path, 'wb+')
 546                 else:
 547                     raise NoSuchMailboxError(self._path)
 548             elif e.errno == errno.EACCES:
 549                 f = open(self._path, 'rb')
 550             else:
 551                 raise
 552         self._file = f
 553         self._toc = None
 554         self._next_key = 0
 555         self._pending = False   # No changes require rewriting the file.
 556         self._locked = False
 557         self._file_length = None        # Used to record mailbox size
 558
 559     def add(self, message):
 560         """Add message and return assigned key."""
 561         self._lookup()
 562         self._toc[self._next_key] = self._append_message(message)
 563         self._next_key += 1
 564         self._pending = True
 565         return self._next_key - 1
 566
 567     def remove(self, key):
 568         """Remove the keyed message; raise KeyError if it doesn't exist."""
 569         self._lookup(key)
 570         del self._toc[key]
 571         self._pending = True
 572
 573     def __setitem__(self, key, message):
 574         """Replace the keyed message; raise KeyError if it doesn't exist."""
 575         self._lookup(key)
 576         self._toc[key] = self._append_message(message)
 577         self._pending = True
 578
 579     def iterkeys(self):
 580         """Return an iterator over keys."""
 581         self._lookup()
 582         for key in self._toc.keys():
 583             yield key
 584
 585     def has_key(self, key):
 586         """Return True if the keyed message exists, False otherwise."""
 587         self._lookup()
 588         return key in self._toc
 589
 590     def __len__(self):
 591         """Return a count of messages in the mailbox."""
 592         self._lookup()
 593         return len(self._toc)
 594
 595     def lock(self):
 596         """Lock the mailbox."""
 597         if not self._locked:
 598             _lock_file(self._file)
 599             self._locked = True
 600
 601     def unlock(self):
 602         """Unlock the mailbox if it is locked."""
 603         if self._locked:
 604             _unlock_file(self._file)
 605             self._locked = False
 606
 607     def flush(self):
 608         """Write any pending changes to disk."""
 609         if not self._pending:
 610             return
 611
 612         # In order to be writing anything out at all, self._toc must
 613         # already have been generated (and presumably has been modified
 614         # by adding or deleting an item).
 615         assert self._toc is not None
 616
 617         # Check length of self._file; if it's changed, some other process
 618         # has modified the mailbox since we scanned it.
 619         self._file.seek(0, 2)
 620         cur_len = self._file.tell()
 621         if cur_len != self._file_length:
 622             raise ExternalClashError('Size of mailbox file changed '
 623                                      '(expected %i, found %i)' %
 624                                      (self._file_length, cur_len))
 625
 626         new_file = _create_temporary(self._path)
 627         try:
 628             new_toc = {}
 629             self._pre_mailbox_hook(new_file)
 630             for key in sorted(self._toc.keys()):
 631                 start, stop = self._toc[key]
 632                 self._file.seek(start)
 633                 self._pre_message_hook(new_file)
 634                 new_start = new_file.tell()
 635                 while True:
 636                     buffer = self._file.read(min(4096,
 637                                                  stop - self._file.tell()))
 638                     if buffer == '':
 639                         break
 640                     new_file.write(buffer)
 641                 new_toc[key] = (new_start, new_file.tell())
 642                 self._post_message_hook(new_file)
 643         except:
 644             new_file.close()
 645             os.remove(new_file.name)
 646             raise
 647         _sync_close(new_file)
 648         # self._file is about to get replaced, so no need to sync.
 649         self._file.close()
 650         try:
 651             os.rename(new_file.name, self._path)
 652         except OSError, e:
 653             if e.errno == errno.EEXIST or \
 654               (os.name == 'os2' and e.errno == errno.EACCES):
 655                 os.remove(self._path)
 656                 os.rename(new_file.name, self._path)
 657             else:
 658                 raise
 659         self._file = open(self._path, 'rb+')
 660         self._toc = new_toc
 661         self._pending = False
 662         if self._locked:
 663             _lock_file(self._file, dotlock=False)
 664
 665     def _pre_mailbox_hook(self, f):
 666         """Called before writing the mailbox to file f."""
 667         return
 668
 669     def _pre_message_hook(self, f):
 670         """Called before writing each message to file f."""
 671         return
 672
 673     def _post_message_hook(self, f):
 674         """Called after writing each message to file f."""
 675         return
 676
 677     def close(self):
 678         """Flush and close the mailbox."""
 679         self.flush()
 680         if self._locked:
 681             self.unlock()
 682         self._file.close()  # Sync has been done by self.flush() above.
 683
 684     def _lookup(self, key=None):
 685         """Return (start, stop) or raise KeyError."""
 686         if self._toc is None:
 687             self._generate_toc()
 688         if key is not None:
 689             try:
 690                 return self._toc[key]
 691             except KeyError:
 692                 raise KeyError('No message with key: %s' % key)
 693
 694     def _append_message(self, message):
 695         """Append message to mailbox and return (start, stop) offsets."""
 696         self._file.seek(0, 2)
 697         self._pre_message_hook(self._file)
 698         offsets = self._install_message(message)
 699         self._post_message_hook(self._file)
 700         self._file.flush()
 701         self._file_length = self._file.tell()  # Record current length of mailbox
 702         return offsets
 703
 704
 705
 706 class _mboxMMDF(_singlefileMailbox):
 707     """An mbox or MMDF mailbox."""
 708
 709     _mangle_from_ = True
 710
 711     def get_message(self, key):
 712         """Return a Message representation or raise a KeyError."""
 713         start, stop = self._lookup(key)
 714         self._file.seek(start)
 715         from_line = self._file.readline().replace(os.linesep, '')
 716         string = self._file.read(stop - self._file.tell())
 717         msg = self._message_factory(string.replace(os.linesep, '\n'))
 718         msg.set_from(from_line[5:])
 719         return msg
 720
 721     def get_string(self, key, from_=False):
 722         """Return a string representation or raise a KeyError."""
 723         start, stop = self._lookup(key)
 724         self._file.seek(start)
 725         if not from_:
 726             self._file.readline()
 727         string = self._file.read(stop - self._file.tell())
 728         return string.replace(os.linesep, '\n')
 729
 730     def get_file(self, key, from_=False):
 731         """Return a file-like representation or raise a KeyError."""
 732         start, stop = self._lookup(key)
 733         self._file.seek(start)
 734         if not from_:
 735             self._file.readline()
 736         return _PartialFile(self._file, self._file.tell(), stop)
 737
 738     def _install_message(self, message):
 739         """Format a message and blindly write to self._file."""
 740         from_line = None
 741         if isinstance(message, str) and message.startswith('From '):
 742             newline = message.find('\n')
 743             if newline != -1:
 744                 from_line = message[:newline]
 745                 message = message[newline + 1:]
 746             else:
 747                 from_line = message
 748                 message = ''
 749         elif isinstance(message, _mboxMMDFMessage):
 750             from_line = 'From ' + message.get_from()
 751         elif isinstance(message, email.message.Message):
 752             from_line = message.get_unixfrom()  # May be None.
 753         if from_line is None:
 754             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 755         start = self._file.tell()
 756         self._file.write(from_line + os.linesep)
 757         self._dump_message(message, self._file, self._mangle_from_)
 758         stop = self._file.tell()
 759         return (start, stop)
 760
 761
 762 class mbox(_mboxMMDF):
 763     """A classic mbox mailbox."""
 764
 765     _mangle_from_ = True
 766
 767     def __init__(self, path, factory=None, create=True):
 768         """Initialize an mbox mailbox."""
 769         self._message_factory = mboxMessage
 770         _mboxMMDF.__init__(self, path, factory, create)
 771
 772     def _pre_message_hook(self, f):
 773         """Called before writing each message to file f."""
 774         if f.tell() != 0:
 775             f.write(os.linesep)
 776
 777     def _generate_toc(self):
 778         """Generate key-to-(start, stop) table of contents."""
 779         starts, stops = [], []
 780         self._file.seek(0)
 781         while True:
 782             line_pos = self._file.tell()
 783             line = self._file.readline()
 784             if line.startswith('From '):
 785                 if len(stops) < len(starts):
 786                     stops.append(line_pos - len(os.linesep))
 787                 starts.append(line_pos)
 788             elif line == '':
 789                 stops.append(line_pos)
 790                 break
 791         self._toc = dict(enumerate(zip(starts, stops)))
 792         self._next_key = len(self._toc)
 793         self._file_length = self._file.tell()
 794
 795
 796 class MMDF(_mboxMMDF):
 797     """An MMDF mailbox."""
 798
 799     def __init__(self, path, factory=None, create=True):
 800         """Initialize an MMDF mailbox."""
 801         self._message_factory = MMDFMessage
 802         _mboxMMDF.__init__(self, path, factory, create)
 803
 804     def _pre_message_hook(self, f):
 805         """Called before writing each message to file f."""
 806         f.write('\001\001\001\001' + os.linesep)
 807
 808     def _post_message_hook(self, f):
 809         """Called after writing each message to file f."""
 810         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 811
 812     def _generate_toc(self):
 813         """Generate key-to-(start, stop) table of contents."""
 814         starts, stops = [], []
 815         self._file.seek(0)
 816         next_pos = 0
 817         while True:
 818             line_pos = next_pos
 819             line = self._file.readline()
 820             next_pos = self._file.tell()
 821             if line.startswith('\001\001\001\001' + os.linesep):
 822                 starts.append(next_pos)
 823                 while True:
 824                     line_pos = next_pos
 825                     line = self._file.readline()
 826                     next_pos = self._file.tell()
 827                     if line == '\001\001\001\001' + os.linesep:
 828                         stops.append(line_pos - len(os.linesep))
 829                         break
 830                     elif line == '':
 831                         stops.append(line_pos)
 832                         break
 833             elif line == '':
 834                 break
 835         self._toc = dict(enumerate(zip(starts, stops)))
 836         self._next_key = len(self._toc)
 837         self._file.seek(0, 2)
 838         self._file_length = self._file.tell()
 839
 840
 841 class MH(Mailbox):
 842     """An MH mailbox."""
 843
 844     def __init__(self, path, factory=None, create=True):
 845         """Initialize an MH instance."""
 846         Mailbox.__init__(self, path, factory, create)
 847         if not os.path.exists(self._path):
 848             if create:
 849                 os.mkdir(self._path, 0700)
 850                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 851                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
 852             else:
 853                 raise NoSuchMailboxError(self._path)
 854         self._locked = False
 855
 856     def add(self, message):
 857         """Add message and return assigned key."""
 858         keys = self.keys()
 859         if len(keys) == 0:
 860             new_key = 1
 861         else:
 862             new_key = max(keys) + 1
 863         new_path = os.path.join(self._path, str(new_key))
 864         f = _create_carefully(new_path)
 865         try:
 866             if self._locked:
 867                 _lock_file(f)
 868             try:
 869                 self._dump_message(message, f)
 870                 if isinstance(message, MHMessage):
 871                     self._dump_sequences(message, new_key)
 872             finally:
 873                 if self._locked:
 874                     _unlock_file(f)
 875         finally:
 876             _sync_close(f)
 877         return new_key
 878
 879     def remove(self, key):
 880         """Remove the keyed message; raise KeyError if it doesn't exist."""
 881         path = os.path.join(self._path, str(key))
 882         try:
 883             f = open(path, 'rb+')
 884         except IOError, e:
 885             if e.errno == errno.ENOENT:
 886                 raise KeyError('No message with key: %s' % key)
 887             else:
 888                 raise
 889         try:
 890             if self._locked:
 891                 _lock_file(f)
 892             try:
 893                 f.close()
 894                 os.remove(os.path.join(self._path, str(key)))
 895             finally:
 896                 if self._locked:
 897                     _unlock_file(f)
 898         finally:
 899             f.close()
 900
 901     def __setitem__(self, key, message):
 902         """Replace the keyed message; raise KeyError if it doesn't exist."""
 903         path = os.path.join(self._path, str(key))
 904         try:
 905             f = open(path, 'rb+')
 906         except IOError, e:
 907             if e.errno == errno.ENOENT:
 908                 raise KeyError('No message with key: %s' % key)
 909             else:
 910                 raise
 911         try:
 912             if self._locked:
 913                 _lock_file(f)
 914             try:
 915                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 916                 self._dump_message(message, f)
 917                 if isinstance(message, MHMessage):
 918                     self._dump_sequences(message, key)
 919             finally:
 920                 if self._locked:
 921                     _unlock_file(f)
 922         finally:
 923             _sync_close(f)
 924
 925     def get_message(self, key):
 926         """Return a Message representation or raise a KeyError."""
 927         try:
 928             if self._locked:
 929                 f = open(os.path.join(self._path, str(key)), 'r+')
 930             else:
 931                 f = open(os.path.join(self._path, str(key)), 'r')
 932         except IOError, e:
 933             if e.errno == errno.ENOENT:
 934                 raise KeyError('No message with key: %s' % key)
 935             else:
 936                 raise
 937         try:
 938             if self._locked:
 939                 _lock_file(f)
 940             try:
 941                 msg = MHMessage(f)
 942             finally:
 943                 if self._locked:
 944                     _unlock_file(f)
 945         finally:
 946             f.close()
 947         for name, key_list in self.get_sequences().iteritems():
 948             if key in key_list:
 949                 msg.add_sequence(name)
 950         return msg
 951
 952     def get_string(self, key):
 953         """Return a string representation or raise a KeyError."""
 954         try:
 955             if self._locked:
 956                 f = open(os.path.join(self._path, str(key)), 'r+')
 957             else:
 958                 f = open(os.path.join(self._path, str(key)), 'r')
 959         except IOError, e:
 960             if e.errno == errno.ENOENT:
 961                 raise KeyError('No message with key: %s' % key)
 962             else:
 963                 raise
 964         try:
 965             if self._locked:
 966                 _lock_file(f)
 967             try:
 968                 return f.read()
 969             finally:
 970                 if self._locked:
 971                     _unlock_file(f)
 972         finally:
 973             f.close()
 974
 975     def get_file(self, key):
 976         """Return a file-like representation or raise a KeyError."""
 977         try:
 978             f = open(os.path.join(self._path, str(key)), 'rb')
 979         except IOError, e:
 980             if e.errno == errno.ENOENT:
 981                 raise KeyError('No message with key: %s' % key)
 982             else:
 983                 raise
 984         return _ProxyFile(f)
 985
 986     def iterkeys(self):
 987         """Return an iterator over keys."""
 988         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 989                                       if entry.isdigit()))
 990
 991     def has_key(self, key):
 992         """Return True if the keyed message exists, False otherwise."""
 993         return os.path.exists(os.path.join(self._path, str(key)))
 994
 995     def __len__(self):
 996         """Return a count of messages in the mailbox."""
 997         return len(list(self.iterkeys()))
 998
 999     def lock(self):
1000         """Lock the mailbox."""
1001         if not self._locked:
1002             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
1003             _lock_file(self._file)
1004             self._locked = True
1005
1006     def unlock(self):
1007         """Unlock the mailbox if it is locked."""
1008         if self._locked:
1009             _unlock_file(self._file)
1010             _sync_close(self._file)
1011             del self._file
1012             self._locked = False
1013
1014     def flush(self):
1015         """Write any pending changes to the disk."""
1016         return
1017
1018     def close(self):
1019         """Flush and close the mailbox."""
1020         if self._locked:
1021             self.unlock()
1022
1023     def list_folders(self):
1024         """Return a list of folder names."""
1025         result = []
1026         for entry in os.listdir(self._path):
1027             if os.path.isdir(os.path.join(self._path, entry)):
1028                 result.append(entry)
1029         return result
1030
1031     def get_folder(self, folder):
1032         """Return an MH instance for the named folder."""
1033         return MH(os.path.join(self._path, folder),
1034                   factory=self._factory, create=False)
1035
1036     def add_folder(self, folder):
1037         """Create a folder and return an MH instance representing it."""
1038         return MH(os.path.join(self._path, folder),
1039                   factory=self._factory)
1040
1041     def remove_folder(self, folder):
1042         """Delete the named folder, which must be empty."""
1043         path = os.path.join(self._path, folder)
1044         entries = os.listdir(path)
1045         if entries == ['.mh_sequences']:
1046             os.remove(os.path.join(path, '.mh_sequences'))
1047         elif entries == []:
1048             pass
1049         else:
1050             raise NotEmptyError('Folder not empty: %s' % self._path)
1051         os.rmdir(path)
1052
1053     def get_sequences(self):
1054         """Return a name-to-key-list dictionary to define each sequence."""
1055         results = {}
1056         f = open(os.path.join(self._path, '.mh_sequences'), 'r')
1057         try:
1058             all_keys = set(self.keys())
1059             for line in f:
1060                 try:
1061                     name, contents = line.split(':')
1062                     keys = set()
1063                     for spec in contents.split():
1064                         if spec.isdigit():
1065                             keys.add(int(spec))
1066                         else:
1067                             start, stop = (int(x) for x in spec.split('-'))
1068                             keys.update(range(start, stop + 1))
1069                     results[name] = [key for key in sorted(keys) \
1070                                          if key in all_keys]
1071                     if len(results[name]) == 0:
1072                         del results[name]
1073                 except ValueError:
1074                     raise FormatError('Invalid sequence specification: %s' %
1075                                       line.rstrip())
1076         finally:
1077             f.close()
1078         return results
1079
1080     def set_sequences(self, sequences):
1081         """Set sequences using the given name-to-key-list dictionary."""
1082         f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
1083         try:
1084             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1085             for name, keys in sequences.iteritems():
1086                 if len(keys) == 0:
1087                     continue
1088                 f.write('%s:' % name)
1089                 prev = None
1090                 completing = False
1091                 for key in sorted(set(keys)):
1092                     if key - 1 == prev:
1093                         if not completing:
1094                             completing = True
1095                             f.write('-')
1096                     elif completing:
1097                         completing = False
1098                         f.write('%s %s' % (prev, key))
1099                     else:
1100                         f.write(' %s' % key)
1101                     prev = key
1102                 if completing:
1103                     f.write(str(prev) + '\n')
1104                 else:
1105                     f.write('\n')
1106         finally:
1107             _sync_close(f)
1108
1109     def pack(self):
1110         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1111         sequences = self.get_sequences()
1112         prev = 0
1113         changes = []
1114         for key in self.iterkeys():
1115             if key - 1 != prev:
1116                 changes.append((key, prev + 1))
1117                 if hasattr(os, 'link'):
1118                     os.link(os.path.join(self._path, str(key)),
1119                             os.path.join(self._path, str(prev + 1)))
1120                     os.unlink(os.path.join(self._path, str(key)))
1121                 else:
1122                     os.rename(os.path.join(self._path, str(key)),
1123                               os.path.join(self._path, str(prev + 1)))
1124             prev += 1
1125         self._next_key = prev + 1
1126         if len(changes) == 0:
1127             return
1128         for name, key_list in sequences.items():
1129             for old, new in changes:
1130                 if old in key_list:
1131                     key_list[key_list.index(old)] = new
1132         self.set_sequences(sequences)
1133
1134     def _dump_sequences(self, message, key):
1135         """Inspect a new MHMessage and update sequences appropriately."""
1136         pending_sequences = message.get_sequences()
1137         all_sequences = self.get_sequences()
1138         for name, key_list in all_sequences.iteritems():
1139             if name in pending_sequences:
1140                 key_list.append(key)
1141             elif key in key_list:
1142                 del key_list[key_list.index(key)]
1143         for sequence in pending_sequences:
1144             if sequence not in all_sequences:
1145                 all_sequences[sequence] = [key]
1146         self.set_sequences(all_sequences)
1147
1148
1149 class Babyl(_singlefileMailbox):
1150     """An Rmail-style Babyl mailbox."""
1151
1152     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1153                                  'forwarded', 'edited', 'resent'))
1154
1155     def __init__(self, path, factory=None, create=True):
1156         """Initialize a Babyl mailbox."""
1157         _singlefileMailbox.__init__(self, path, factory, create)
1158         self._labels = {}
1159
1160     def add(self, message):
1161         """Add message and return assigned key."""
1162         key = _singlefileMailbox.add(self, message)
1163         if isinstance(message, BabylMessage):
1164             self._labels[key] = message.get_labels()
1165         return key
1166
1167     def remove(self, key):
1168         """Remove the keyed message; raise KeyError if it doesn't exist."""
1169         _singlefileMailbox.remove(self, key)
1170         if key in self._labels:
1171             del self._labels[key]
1172
1173     def __setitem__(self, key, message):
1174         """Replace the keyed message; raise KeyError if it doesn't exist."""
1175         _singlefileMailbox.__setitem__(self, key, message)
1176         if isinstance(message, BabylMessage):
1177             self._labels[key] = message.get_labels()
1178
1179     def get_message(self, key):
1180         """Return a Message representation or raise a KeyError."""
1181         start, stop = self._lookup(key)
1182         self._file.seek(start)
1183         self._file.readline()   # Skip '1,' line specifying labels.
1184         original_headers = StringIO.StringIO()
1185         while True:
1186             line = self._file.readline()
1187             if line == '*** EOOH ***' + os.linesep or line == '':
1188                 break
1189             original_headers.write(line.replace(os.linesep, '\n'))
1190         visible_headers = StringIO.StringIO()
1191         while True:
1192             line = self._file.readline()
1193             if line == os.linesep or line == '':
1194                 break
1195             visible_headers.write(line.replace(os.linesep, '\n'))
1196         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1197                                                                  '\n')
1198         msg = BabylMessage(original_headers.getvalue() + body)
1199         msg.set_visible(visible_headers.getvalue())
1200         if key in self._labels:
1201             msg.set_labels(self._labels[key])
1202         return msg
1203
1204     def get_string(self, key):
1205         """Return a string representation or raise a KeyError."""
1206         start, stop = self._lookup(key)
1207         self._file.seek(start)
1208         self._file.readline()   # Skip '1,' line specifying labels.
1209         original_headers = StringIO.StringIO()
1210         while True:
1211             line = self._file.readline()
1212             if line == '*** EOOH ***' + os.linesep or line == '':
1213                 break
1214             original_headers.write(line.replace(os.linesep, '\n'))
1215         while True:
1216             line = self._file.readline()
1217             if line == os.linesep or line == '':
1218                 break
1219         return original_headers.getvalue() + \
1220                self._file.read(stop - self._file.tell()).replace(os.linesep,
1221                                                                  '\n')
1222
1223     def get_file(self, key):
1224         """Return a file-like representation or raise a KeyError."""
1225         return StringIO.StringIO(self.get_string(key).replace('\n',
1226                                                               os.linesep))
1227
1228     def get_labels(self):
1229         """Return a list of user-defined labels in the mailbox."""
1230         self._lookup()
1231         labels = set()
1232         for label_list in self._labels.values():
1233             labels.update(label_list)
1234         labels.difference_update(self._special_labels)
1235         return list(labels)
1236
1237     def _generate_toc(self):
1238         """Generate key-to-(start, stop) table of contents."""
1239         starts, stops = [], []
1240         self._file.seek(0)
1241         next_pos = 0
1242         label_lists = []
1243         while True:
1244             line_pos = next_pos
1245             line = self._file.readline()
1246             next_pos = self._file.tell()
1247             if line == '\037\014' + os.linesep:
1248                 if len(stops) < len(starts):
1249                     stops.append(line_pos - len(os.linesep))
1250                 starts.append(next_pos)
1251                 labels = [label.strip() for label
1252                                         in self._file.readline()[1:].split(',')
1253                                         if label.strip() != '']
1254                 label_lists.append(labels)
1255             elif line == '\037' or line == '\037' + os.linesep:
1256                 if len(stops) < len(starts):
1257                     stops.append(line_pos - len(os.linesep))
1258             elif line == '':
1259                 stops.append(line_pos - len(os.linesep))
1260                 break
1261         self._toc = dict(enumerate(zip(starts, stops)))
1262         self._labels = dict(enumerate(label_lists))
1263         self._next_key = len(self._toc)
1264         self._file.seek(0, 2)
1265         self._file_length = self._file.tell()
1266
1267     def _pre_mailbox_hook(self, f):
1268         """Called before writing the mailbox to file f."""
1269         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1270                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1271                  os.linesep))
1272
1273     def _pre_message_hook(self, f):
1274         """Called before writing each message to file f."""
1275         f.write('\014' + os.linesep)
1276
1277     def _post_message_hook(self, f):
1278         """Called after writing each message to file f."""
1279         f.write(os.linesep + '\037')
1280
1281     def _install_message(self, message):
1282         """Write message contents and return (start, stop)."""
1283         start = self._file.tell()
1284         if isinstance(message, BabylMessage):
1285             special_labels = []
1286             labels = []
1287             for label in message.get_labels():
1288                 if label in self._special_labels:
1289                     special_labels.append(label)
1290                 else:
1291                     labels.append(label)
1292             self._file.write('1')
1293             for label in special_labels:
1294                 self._file.write(', ' + label)
1295             self._file.write(',,')
1296             for label in labels:
1297                 self._file.write(' ' + label + ',')
1298             self._file.write(os.linesep)
1299         else:
1300             self._file.write('1,,' + os.linesep)
1301         if isinstance(message, email.message.Message):
1302             orig_buffer = StringIO.StringIO()
1303             orig_generator = email.generator.Generator(orig_buffer, False, 0)
1304             orig_generator.flatten(message)
1305             orig_buffer.seek(0)
1306             while True:
1307                 line = orig_buffer.readline()
1308                 self._file.write(line.replace('\n', os.linesep))
1309                 if line == '\n' or line == '':
1310                     break
1311             self._file.write('*** EOOH ***' + os.linesep)
1312             if isinstance(message, BabylMessage):
1313                 vis_buffer = StringIO.StringIO()
1314                 vis_generator = email.generator.Generator(vis_buffer, False, 0)
1315                 vis_generator.flatten(message.get_visible())
1316                 while True:
1317                     line = vis_buffer.readline()
1318                     self._file.write(line.replace('\n', os.linesep))
1319                     if line == '\n' or line == '':
1320                         break
1321             else:
1322                 orig_buffer.seek(0)
1323                 while True:
1324                     line = orig_buffer.readline()
1325                     self._file.write(line.replace('\n', os.linesep))
1326                     if line == '\n' or line == '':
1327                         break
1328             while True:
1329                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1330                 if buffer == '':
1331                     break
1332                 self._file.write(buffer.replace('\n', os.linesep))
1333         elif isinstance(message, str):
1334             body_start = message.find('\n\n') + 2
1335             if body_start - 2 != -1:
1336                 self._file.write(message[:body_start].replace('\n',
1337                                                               os.linesep))
1338                 self._file.write('*** EOOH ***' + os.linesep)
1339                 self._file.write(message[:body_start].replace('\n',
1340                                                               os.linesep))
1341                 self._file.write(message[body_start:].replace('\n',
1342                                                               os.linesep))
1343             else:
1344                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1345                 self._file.write(message.replace('\n', os.linesep))
1346         elif hasattr(message, 'readline'):
1347             original_pos = message.tell()
1348             first_pass = True
1349             while True:
1350                 line = message.readline()
1351                 self._file.write(line.replace('\n', os.linesep))
1352                 if line == '\n' or line == '':
1353                     self._file.write('*** EOOH ***' + os.linesep)
1354                     if first_pass:
1355                         first_pass = False
1356                         message.seek(original_pos)
1357                     else:
1358                         break
1359             while True:
1360                 buffer = message.read(4096)     # Buffer size is arbitrary.
1361                 if buffer == '':
1362                     break
1363                 self._file.write(buffer.replace('\n', os.linesep))
1364         else:
1365             raise TypeError('Invalid message type: %s' % type(message))
1366         stop = self._file.tell()
1367         return (start, stop)
1368
1369
1370 class Message(email.message.Message):
1371     """Message with mailbox-format-specific properties."""
1372
1373     def __init__(self, message=None):
1374         """Initialize a Message instance."""
1375         if isinstance(message, email.message.Message):
1376             self._become_message(copy.deepcopy(message))
1377             if isinstance(message, Message):
1378                 message._explain_to(self)
1379         elif isinstance(message, str):
1380             self._become_message(email.message_from_string(message))
1381         elif hasattr(message, "read"):
1382             self._become_message(email.message_from_file(message))
1383         elif message is None:
1384             email.message.Message.__init__(self)
1385         else:
1386             raise TypeError('Invalid message type: %s' % type(message))
1387
1388     def _become_message(self, message):
1389         """Assume the non-format-specific state of message."""
1390         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1391                      'preamble', 'epilogue', 'defects', '_default_type'):
1392             self.__dict__[name] = message.__dict__[name]
1393
1394     def _explain_to(self, message):
1395         """Copy format-specific state to message insofar as possible."""
1396         if isinstance(message, Message):
1397             return  # There's nothing format-specific to explain.
1398         else:
1399             raise TypeError('Cannot convert to specified type')
1400
1401
1402 class MaildirMessage(Message):
1403     """Message with Maildir-specific properties."""
1404
1405     def __init__(self, message=None):
1406         """Initialize a MaildirMessage instance."""
1407         self._subdir = 'new'
1408         self._info = ''
1409         self._date = time.time()
1410         Message.__init__(self, message)
1411
1412     def get_subdir(self):
1413         """Return 'new' or 'cur'."""
1414         return self._subdir
1415
1416     def set_subdir(self, subdir):
1417         """Set subdir to 'new' or 'cur'."""
1418         if subdir == 'new' or subdir == 'cur':
1419             self._subdir = subdir
1420         else:
1421             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1422
1423     def get_flags(self):
1424         """Return as a string the flags that are set."""
1425         if self._info.startswith('2,'):
1426             return self._info[2:]
1427         else:
1428             return ''
1429
1430     def set_flags(self, flags):
1431         """Set the given flags and unset all others."""
1432         self._info = '2,' + ''.join(sorted(flags))
1433
1434     def add_flag(self, flag):
1435         """Set the given flag(s) without changing others."""
1436         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1437
1438     def remove_flag(self, flag):
1439         """Unset the given string flag(s) without changing others."""
1440         if self.get_flags() != '':
1441             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1442
1443     def get_date(self):
1444         """Return delivery date of message, in seconds since the epoch."""
1445         return self._date
1446
1447     def set_date(self, date):
1448         """Set delivery date of message, in seconds since the epoch."""
1449         try:
1450             self._date = float(date)
1451         except ValueError:
1452             raise TypeError("can't convert to float: %s" % date)
1453
1454     def get_info(self):
1455         """Get the message's "info" as a string."""
1456         return self._info
1457
1458     def set_info(self, info):
1459         """Set the message's "info" string."""
1460         if isinstance(info, str):
1461             self._info = info
1462         else:
1463             raise TypeError('info must be a string: %s' % type(info))
1464
1465     def _explain_to(self, message):
1466         """Copy Maildir-specific state to message insofar as possible."""
1467         if isinstance(message, MaildirMessage):
1468             message.set_flags(self.get_flags())
1469             message.set_subdir(self.get_subdir())
1470             message.set_date(self.get_date())
1471         elif isinstance(message, _mboxMMDFMessage):
1472             flags = set(self.get_flags())
1473             if 'S' in flags:
1474                 message.add_flag('R')
1475             if self.get_subdir() == 'cur':
1476                 message.add_flag('O')
1477             if 'T' in flags:
1478                 message.add_flag('D')
1479             if 'F' in flags:
1480                 message.add_flag('F')
1481             if 'R' in flags:
1482                 message.add_flag('A')
1483             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1484         elif isinstance(message, MHMessage):
1485             flags = set(self.get_flags())
1486             if 'S' not in flags:
1487                 message.add_sequence('unseen')
1488             if 'R' in flags:
1489                 message.add_sequence('replied')
1490             if 'F' in flags:
1491                 message.add_sequence('flagged')
1492         elif isinstance(message, BabylMessage):
1493             flags = set(self.get_flags())
1494             if 'S' not in flags:
1495                 message.add_label('unseen')
1496             if 'T' in flags:
1497                 message.add_label('deleted')
1498             if 'R' in flags:
1499                 message.add_label('answered')
1500             if 'P' in flags:
1501                 message.add_label('forwarded')
1502         elif isinstance(message, Message):
1503             pass
1504         else:
1505             raise TypeError('Cannot convert to specified type: %s' %
1506                             type(message))
1507
1508
1509 class _mboxMMDFMessage(Message):
1510     """Message with mbox- or MMDF-specific properties."""
1511
1512     def __init__(self, message=None):
1513         """Initialize an mboxMMDFMessage instance."""
1514         self.set_from('MAILER-DAEMON', True)
1515         if isinstance(message, email.message.Message):
1516             unixfrom = message.get_unixfrom()
1517             if unixfrom is not None and unixfrom.startswith('From '):
1518                 self.set_from(unixfrom[5:])
1519         Message.__init__(self, message)
1520
1521     def get_from(self):
1522         """Return contents of "From " line."""
1523         return self._from
1524
1525     def set_from(self, from_, time_=None):
1526         """Set "From " line, formatting and appending time_ if specified."""
1527         if time_ is not None:
1528             if time_ is True:
1529                 time_ = time.gmtime()
1530             from_ += ' ' + time.asctime(time_)
1531         self._from = from_
1532
1533     def get_flags(self):
1534         """Return as a string the flags that are set."""
1535         return self.get('Status', '') + self.get('X-Status', '')
1536
1537     def set_flags(self, flags):
1538         """Set the given flags and unset all others."""
1539         flags = set(flags)
1540         status_flags, xstatus_flags = '', ''
1541         for flag in ('R', 'O'):
1542             if flag in flags:
1543                 status_flags += flag
1544                 flags.remove(flag)
1545         for flag in ('D', 'F', 'A'):
1546             if flag in flags:
1547                 xstatus_flags += flag
1548                 flags.remove(flag)
1549         xstatus_flags += ''.join(sorted(flags))
1550         try:
1551             self.replace_header('Status', status_flags)
1552         except KeyError:
1553             self.add_header('Status', status_flags)
1554         try:
1555             self.replace_header('X-Status', xstatus_flags)
1556         except KeyError:
1557             self.add_header('X-Status', xstatus_flags)
1558
1559     def add_flag(self, flag):
1560         """Set the given flag(s) without changing others."""
1561         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1562
1563     def remove_flag(self, flag):
1564         """Unset the given string flag(s) without changing others."""
1565         if 'Status' in self or 'X-Status' in self:
1566             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1567
1568     def _explain_to(self, message):
1569         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1570         if isinstance(message, MaildirMessage):
1571             flags = set(self.get_flags())
1572             if 'O' in flags:
1573                 message.set_subdir('cur')
1574             if 'F' in flags:
1575                 message.add_flag('F')
1576             if 'A' in flags:
1577                 message.add_flag('R')
1578             if 'R' in flags:
1579                 message.add_flag('S')
1580             if 'D' in flags:
1581                 message.add_flag('T')
1582             del message['status']
1583             del message['x-status']
1584             maybe_date = ' '.join(self.get_from().split()[-5:])
1585             try:
1586                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1587                                                       '%a %b %d %H:%M:%S %Y')))
1588             except (ValueError, OverflowError):
1589                 pass
1590         elif isinstance(message, _mboxMMDFMessage):
1591             message.set_flags(self.get_flags())
1592             message.set_from(self.get_from())
1593         elif isinstance(message, MHMessage):
1594             flags = set(self.get_flags())
1595             if 'R' not in flags:
1596                 message.add_sequence('unseen')
1597             if 'A' in flags:
1598                 message.add_sequence('replied')
1599             if 'F' in flags:
1600                 message.add_sequence('flagged')
1601             del message['status']
1602             del message['x-status']
1603         elif isinstance(message, BabylMessage):
1604             flags = set(self.get_flags())
1605             if 'R' not in flags:
1606                 message.add_label('unseen')
1607             if 'D' in flags:
1608                 message.add_label('deleted')
1609             if 'A' in flags:
1610                 message.add_label('answered')
1611             del message['status']
1612             del message['x-status']
1613         elif isinstance(message, Message):
1614             pass
1615         else:
1616             raise TypeError('Cannot convert to specified type: %s' %
1617                             type(message))
1618
1619
1620 class mboxMessage(_mboxMMDFMessage):
1621     """Message with mbox-specific properties."""
1622
1623
1624 class MHMessage(Message):
1625     """Message with MH-specific properties."""
1626
1627     def __init__(self, message=None):
1628         """Initialize an MHMessage instance."""
1629         self._sequences = []
1630         Message.__init__(self, message)
1631
1632     def get_sequences(self):
1633         """Return a list of sequences that include the message."""
1634         return self._sequences[:]
1635
1636     def set_sequences(self, sequences):
1637         """Set the list of sequences that include the message."""
1638         self._sequences = list(sequences)
1639
1640     def add_sequence(self, sequence):
1641         """Add sequence to list of sequences including the message."""
1642         if isinstance(sequence, str):
1643             if not sequence in self._sequences:
1644                 self._sequences.append(sequence)
1645         else:
1646             raise TypeError('sequence must be a string: %s' % type(sequence))
1647
1648     def remove_sequence(self, sequence):
1649         """Remove sequence from the list of sequences including the message."""
1650         try:
1651             self._sequences.remove(sequence)
1652         except ValueError:
1653             pass
1654
1655     def _explain_to(self, message):
1656         """Copy MH-specific state to message insofar as possible."""
1657         if isinstance(message, MaildirMessage):
1658             sequences = set(self.get_sequences())
1659             if 'unseen' in sequences:
1660                 message.set_subdir('cur')
1661             else:
1662                 message.set_subdir('cur')
1663                 message.add_flag('S')
1664             if 'flagged' in sequences:
1665                 message.add_flag('F')
1666             if 'replied' in sequences:
1667                 message.add_flag('R')
1668         elif isinstance(message, _mboxMMDFMessage):
1669             sequences = set(self.get_sequences())
1670             if 'unseen' not in sequences:
1671                 message.add_flag('RO')
1672             else:
1673                 message.add_flag('O')
1674             if 'flagged' in sequences:
1675                 message.add_flag('F')
1676             if 'replied' in sequences:
1677                 message.add_flag('A')
1678         elif isinstance(message, MHMessage):
1679             for sequence in self.get_sequences():
1680                 message.add_sequence(sequence)
1681         elif isinstance(message, BabylMessage):
1682             sequences = set(self.get_sequences())
1683             if 'unseen' in sequences:
1684                 message.add_label('unseen')
1685             if 'replied' in sequences:
1686                 message.add_label('answered')
1687         elif isinstance(message, Message):
1688             pass
1689         else:
1690             raise TypeError('Cannot convert to specified type: %s' %
1691                             type(message))
1692
1693
1694 class BabylMessage(Message):
1695     """Message with Babyl-specific properties."""
1696
1697     def __init__(self, message=None):
1698         """Initialize an BabylMessage instance."""
1699         self._labels = []
1700         self._visible = Message()
1701         Message.__init__(self, message)
1702
1703     def get_labels(self):
1704         """Return a list of labels on the message."""
1705         return self._labels[:]
1706
1707     def set_labels(self, labels):
1708         """Set the list of labels on the message."""
1709         self._labels = list(labels)
1710
1711     def add_label(self, label):
1712         """Add label to list of labels on the message."""
1713         if isinstance(label, str):
1714             if label not in self._labels:
1715                 self._labels.append(label)
1716         else:
1717             raise TypeError('label must be a string: %s' % type(label))
1718
1719     def remove_label(self, label):
1720         """Remove label from the list of labels on the message."""
1721         try:
1722             self._labels.remove(label)
1723         except ValueError:
1724             pass
1725
1726     def get_visible(self):
1727         """Return a Message representation of visible headers."""
1728         return Message(self._visible)
1729
1730     def set_visible(self, visible):
1731         """Set the Message representation of visible headers."""
1732         self._visible = Message(visible)
1733
1734     def update_visible(self):
1735         """Update and/or sensibly generate a set of visible headers."""
1736         for header in self._visible.keys():
1737             if header in self:
1738                 self._visible.replace_header(header, self[header])
1739             else:
1740                 del self._visible[header]
1741         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1742             if header in self and header not in self._visible:
1743                 self._visible[header] = self[header]
1744
1745     def _explain_to(self, message):
1746         """Copy Babyl-specific state to message insofar as possible."""
1747         if isinstance(message, MaildirMessage):
1748             labels = set(self.get_labels())
1749             if 'unseen' in labels:
1750                 message.set_subdir('cur')
1751             else:
1752                 message.set_subdir('cur')
1753                 message.add_flag('S')
1754             if 'forwarded' in labels or 'resent' in labels:
1755                 message.add_flag('P')
1756             if 'answered' in labels:
1757                 message.add_flag('R')
1758             if 'deleted' in labels:
1759                 message.add_flag('T')
1760         elif isinstance(message, _mboxMMDFMessage):
1761             labels = set(self.get_labels())
1762             if 'unseen' not in labels:
1763                 message.add_flag('RO')
1764             else:
1765                 message.add_flag('O')
1766             if 'deleted' in labels:
1767                 message.add_flag('D')
1768             if 'answered' in labels:
1769                 message.add_flag('A')
1770         elif isinstance(message, MHMessage):
1771             labels = set(self.get_labels())
1772             if 'unseen' in labels:
1773                 message.add_sequence('unseen')
1774             if 'answered' in labels:
1775                 message.add_sequence('replied')
1776         elif isinstance(message, BabylMessage):
1777             message.set_visible(self.get_visible())
1778             for label in self.get_labels():
1779                 message.add_label(label)
1780         elif isinstance(message, Message):
1781             pass
1782         else:
1783             raise TypeError('Cannot convert to specified type: %s' %
1784                             type(message))
1785
1786
1787 class MMDFMessage(_mboxMMDFMessage):
1788     """Message with MMDF-specific properties."""
1789
1790
1791 class _ProxyFile:
1792     """A read-only wrapper of a file."""
1793
1794     def __init__(self, f, pos=None):
1795         """Initialize a _ProxyFile."""
1796         self._file = f
1797         if pos is None:
1798             self._pos = f.tell()
1799         else:
1800             self._pos = pos
1801
1802     def read(self, size=None):
1803         """Read bytes."""
1804         return self._read(size, self._file.read)
1805
1806     def readline(self, size=None):
1807         """Read a line."""
1808         return self._read(size, self._file.readline)
1809
1810     def readlines(self, sizehint=None):
1811         """Read multiple lines."""
1812         result = []
1813         for line in self:
1814             result.append(line)
1815             if sizehint is not None:
1816                 sizehint -= len(line)
1817                 if sizehint <= 0:
1818                     break
1819         return result
1820
1821     def __iter__(self):
1822         """Iterate over lines."""
1823         return iter(self.readline, "")
1824
1825     def tell(self):
1826         """Return the position."""
1827         return self._pos
1828
1829     def seek(self, offset, whence=0):
1830         """Change position."""
1831         if whence == 1:
1832             self._file.seek(self._pos)
1833         self._file.seek(offset, whence)
1834         self._pos = self._file.tell()
1835
1836     def close(self):
1837         """Close the file."""
1838         del self._file
1839
1840     def _read(self, size, read_method):
1841         """Read size bytes using read_method."""
1842         if size is None:
1843             size = -1
1844         self._file.seek(self._pos)
1845         result = read_method(size)
1846         self._pos = self._file.tell()
1847         return result
1848
1849
1850 class _PartialFile(_ProxyFile):
1851     """A read-only wrapper of part of a file."""
1852
1853     def __init__(self, f, start=None, stop=None):
1854         """Initialize a _PartialFile."""
1855         _ProxyFile.__init__(self, f, start)
1856         self._start = start
1857         self._stop = stop
1858
1859     def tell(self):
1860         """Return the position with respect to start."""
1861         return _ProxyFile.tell(self) - self._start
1862
1863     def seek(self, offset, whence=0):
1864         """Change position, possibly with respect to start or stop."""
1865         if whence == 0:
1866             self._pos = self._start
1867             whence = 1
1868         elif whence == 2:
1869             self._pos = self._stop
1870             whence = 1
1871         _ProxyFile.seek(self, offset, whence)
1872
1873     def _read(self, size, read_method):
1874         """Read size bytes using read_method, honoring start and stop."""
1875         remaining = self._stop - self._pos
1876         if remaining <= 0:
1877             return ''
1878         if size is None or size < 0 or size > remaining:
1879             size = remaining
1880         return _ProxyFile._read(self, size, read_method)
1881
1882
1883 def _lock_file(f, dotlock=True):
1884     """Lock file f using lockf and dot locking."""
1885     dotlock_done = False
1886     try:
1887         if fcntl:
1888             try:
1889                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1890             except IOError, e:
1891                 if e.errno in (errno.EAGAIN, errno.EACCES):
1892                     raise ExternalClashError('lockf: lock unavailable: %s' %
1893                                              f.name)
1894                 else:
1895                     raise
1896         if dotlock:
1897             try:
1898                 pre_lock = _create_temporary(f.name + '.lock')
1899                 pre_lock.close()
1900             except IOError, e:
1901                 if e.errno == errno.EACCES:
1902                     return  # Without write access, just skip dotlocking.
1903                 else:
1904                     raise
1905             try:
1906                 if hasattr(os, 'link'):
1907                     os.link(pre_lock.name, f.name + '.lock')
1908                     dotlock_done = True
1909                     os.unlink(pre_lock.name)
1910                 else:
1911                     os.rename(pre_lock.name, f.name + '.lock')
1912                     dotlock_done = True
1913             except OSError, e:
1914                 if e.errno == errno.EEXIST or \
1915                   (os.name == 'os2' and e.errno == errno.EACCES):
1916                     os.remove(pre_lock.name)
1917                     raise ExternalClashError('dot lock unavailable: %s' %
1918                                              f.name)
1919                 else:
1920                     raise
1921     except:
1922         if fcntl:
1923             fcntl.lockf(f, fcntl.LOCK_UN)
1924         if dotlock_done:
1925             os.remove(f.name + '.lock')
1926         raise
1927
1928 def _unlock_file(f):
1929     """Unlock file f using lockf and dot locking."""
1930     if fcntl:
1931         fcntl.lockf(f, fcntl.LOCK_UN)
1932     if os.path.exists(f.name + '.lock'):
1933         os.remove(f.name + '.lock')
1934
1935 def _create_carefully(path):
1936     """Create a file if it doesn't exist and open for reading and writing."""
1937     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
1938     try:
1939         return open(path, 'rb+')
1940     finally:
1941         os.close(fd)
1942
1943 def _create_temporary(path):
1944     """Create a temp file based on path and open for reading and writing."""
1945     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1946                                               socket.gethostname(),
1947                                               os.getpid()))
1948
1949 def _sync_flush(f):
1950     """Ensure changes to file f are physically on disk."""
1951     f.flush()
1952     if hasattr(os, 'fsync'):
1953         os.fsync(f.fileno())
1954
1955 def _sync_close(f):
1956     """Close file f, ensuring all changes are physically on disk."""
1957     _sync_flush(f)
1958     f.close()
1959
1960 ## Start: classes from the original module (for backward compatibility).
1961
1962 # Note that the Maildir class, whose name is unchanged, itself offers a next()
1963 # method for backward compatibility.
1964
1965 class _Mailbox:
1966
1967     def __init__(self, fp, factory=rfc822.Message):
1968         self.fp = fp
1969         self.seekp = 0
1970         self.factory = factory
1971
1972     def __iter__(self):
1973         return iter(self.next, None)
1974
1975     def next(self):
1976         while 1:
1977             self.fp.seek(self.seekp)
1978             try:
1979                 self._search_start()
1980             except EOFError:
1981                 self.seekp = self.fp.tell()
1982                 return None
1983             start = self.fp.tell()
1984             self._search_end()
1985             self.seekp = stop = self.fp.tell()
1986             if start != stop:
1987                 break
1988         return self.factory(_PartialFile(self.fp, start, stop))
1989
1990 # Recommended to use PortableUnixMailbox instead!
1991 class UnixMailbox(_Mailbox):
1992
1993     def _search_start(self):
1994         while 1:
1995             pos = self.fp.tell()
1996             line = self.fp.readline()
1997             if not line:
1998                 raise EOFError
1999             if line[:5] == 'From ' and self._isrealfromline(line):
2000                 self.fp.seek(pos)
2001                 return
2002
2003     def _search_end(self):
2004         self.fp.readline()      # Throw away header line
2005         while 1:
2006             pos = self.fp.tell()
2007             line = self.fp.readline()
2008             if not line:
2009                 return
2010             if line[:5] == 'From ' and self._isrealfromline(line):
2011                 self.fp.seek(pos)
2012                 return
2013
2014     # An overridable mechanism to test for From-line-ness.  You can either
2015     # specify a different regular expression or define a whole new
2016     # _isrealfromline() method.  Note that this only gets called for lines
2017     # starting with the 5 characters "From ".
2018     #
2019     # BAW: According to
2020     #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
2021     # the only portable, reliable way to find message delimiters in a BSD (i.e
2022     # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
2023     # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
2024     # like a good idea, in practice, there are too many variations for more
2025     # strict parsing of the line to be completely accurate.
2026     #
2027     # _strict_isrealfromline() is the old version which tries to do stricter
2028     # parsing of the From_ line.  _portable_isrealfromline() simply returns
2029     # true, since it's never called if the line doesn't already start with
2030     # "From ".
2031     #
2032     # This algorithm, and the way it interacts with _search_start() and
2033     # _search_end() may not be completely correct, because it doesn't check
2034     # that the two characters preceding "From " are \n\n or the beginning of
2035     # the file.  Fixing this would require a more extensive rewrite than is
2036     # necessary.  For convenience, we've added a PortableUnixMailbox class
2037     # which does no checking of the format of the 'From' line.
2038
2039     _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
2040                         r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
2041                         r"[^\s]*\s*"
2042                         "$")
2043     _regexp = None
2044
2045     def _strict_isrealfromline(self, line):
2046         if not self._regexp:
2047             import re
2048             self._regexp = re.compile(self._fromlinepattern)
2049         return self._regexp.match(line)
2050
2051     def _portable_isrealfromline(self, line):
2052         return True
2053
2054     _isrealfromline = _strict_isrealfromline
2055
2056
2057 class PortableUnixMailbox(UnixMailbox):
2058     _isrealfromline = UnixMailbox._portable_isrealfromline
2059
2060
2061 class MmdfMailbox(_Mailbox):
2062
2063     def _search_start(self):
2064         while 1:
2065             line = self.fp.readline()
2066             if not line:
2067                 raise EOFError
2068             if line[:5] == '\001\001\001\001\n':
2069                 return
2070
2071     def _search_end(self):
2072         while 1:
2073             pos = self.fp.tell()
2074             line = self.fp.readline()
2075             if not line:
2076                 return
2077             if line == '\001\001\001\001\n':
2078                 self.fp.seek(pos)
2079                 return
2080
2081
2082 class MHMailbox:
2083
2084     def __init__(self, dirname, factory=rfc822.Message):
2085         import re
2086         pat = re.compile('^[1-9][0-9]*$')
2087         self.dirname = dirname
2088         # the three following lines could be combined into:
2089         # list = map(long, filter(pat.match, os.listdir(self.dirname)))
2090         list = os.listdir(self.dirname)
2091         list = filter(pat.match, list)
2092         list = map(long, list)
2093         list.sort()
2094         # This only works in Python 1.6 or later;
2095         # before that str() added 'L':
2096         self.boxes = map(str, list)
2097         self.boxes.reverse()
2098         self.factory = factory
2099
2100     def __iter__(self):
2101         return iter(self.next, None)
2102
2103     def next(self):
2104         if not self.boxes:
2105             return None
2106         fn = self.boxes.pop()
2107         fp = open(os.path.join(self.dirname, fn))
2108         msg = self.factory(fp)
2109         try:
2110             msg._mh_msgno = fn
2111         except (AttributeError, TypeError):
2112             pass
2113         return msg
2114
2115
2116 class BabylMailbox(_Mailbox):
2117
2118     def _search_start(self):
2119         while 1:
2120             line = self.fp.readline()
2121             if not line:
2122                 raise EOFError
2123             if line == '*** EOOH ***\n':
2124                 return
2125
2126     def _search_end(self):
2127         while 1:
2128             pos = self.fp.tell()
2129             line = self.fp.readline()
2130             if not line:
2131                 return
2132             if line == '\037\014\n' or line == '\037':
2133                 self.fp.seek(pos)
2134                 return
2135
2136 ## End: classes from the original module (for backward compatibility).
2137
2138
2139 class Error(Exception):
2140     """Raised for module-specific errors."""
2141
2142 class NoSuchMailboxError(Error):
2143     """The specified mailbox does not exist and won't be created."""
2144
2145 class NotEmptyError(Error):
2146     """The specified mailbox is not empty and deletion was requested."""
2147
2148 class ExternalClashError(Error):
2149     """Another process caused an action to fail."""
2150
2151 class FormatError(Error):
2152     """A file appears to have an invalid format."""