Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.message
  20 import email.generator
  21 import StringIO
  22 try:
  23     if sys.platform == 'os2emx':
  24         # OS/2 EMX fcntl() not adequate
  25         raise ImportError
  26     import fcntl
  27 except ImportError:
  28     fcntl = None
  29
  30 import warnings
  31 with warnings.catch_warnings():
  32     if sys.py3kwarning:
  33         warnings.filterwarnings("ignore", ".*rfc822 has been removed",
  34                                 DeprecationWarning)
  35     import rfc822
  36
  37 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  38             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  39             'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  40             'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  41
  42 class Mailbox:
  43     """A group of messages in a particular place."""
  44
  45     def __init__(self, path, factory=None, create=True):
  46         """Initialize a Mailbox instance."""
  47         self._path = os.path.abspath(os.path.expanduser(path))
  48         self._factory = factory
  49
  50     def add(self, message):
  51         """Add message and return assigned key."""
  52         raise NotImplementedError('Method must be implemented by subclass')
  53
  54     def remove(self, key):
  55         """Remove the keyed message; raise KeyError if it doesn't exist."""
  56         raise NotImplementedError('Method must be implemented by subclass')
  57
  58     def __delitem__(self, key):
  59         self.remove(key)
  60
  61     def discard(self, key):
  62         """If the keyed message exists, remove it."""
  63         try:
  64             self.remove(key)
  65         except KeyError:
  66             pass
  67
  68     def __setitem__(self, key, message):
  69         """Replace the keyed message; raise KeyError if it doesn't exist."""
  70         raise NotImplementedError('Method must be implemented by subclass')
  71
  72     def get(self, key, default=None):
  73         """Return the keyed message, or default if it doesn't exist."""
  74         try:
  75             return self.__getitem__(key)
  76         except KeyError:
  77             return default
  78
  79     def __getitem__(self, key):
  80         """Return the keyed message; raise KeyError if it doesn't exist."""
  81         if not self._factory:
  82             return self.get_message(key)
  83         else:
  84             return self._factory(self.get_file(key))
  85
  86     def get_message(self, key):
  87         """Return a Message representation or raise a KeyError."""
  88         raise NotImplementedError('Method must be implemented by subclass')
  89
  90     def get_string(self, key):
  91         """Return a string representation or raise a KeyError."""
  92         raise NotImplementedError('Method must be implemented by subclass')
  93
  94     def get_file(self, key):
  95         """Return a file-like representation or raise a KeyError."""
  96         raise NotImplementedError('Method must be implemented by subclass')
  97
  98     def iterkeys(self):
  99         """Return an iterator over keys."""
 100         raise NotImplementedError('Method must be implemented by subclass')
 101
 102     def keys(self):
 103         """Return a list of keys."""
 104         return list(self.iterkeys())
 105
 106     def itervalues(self):
 107         """Return an iterator over all messages."""
 108         for key in self.iterkeys():
 109             try:
 110                 value = self[key]
 111             except KeyError:
 112                 continue
 113             yield value
 114
 115     def __iter__(self):
 116         return self.itervalues()
 117
 118     def values(self):
 119         """Return a list of messages. Memory intensive."""
 120         return list(self.itervalues())
 121
 122     def iteritems(self):
 123         """Return an iterator over (key, message) tuples."""
 124         for key in self.iterkeys():
 125             try:
 126                 value = self[key]
 127             except KeyError:
 128                 continue
 129             yield (key, value)
 130
 131     def items(self):
 132         """Return a list of (key, message) tuples. Memory intensive."""
 133         return list(self.iteritems())
 134
 135     def has_key(self, key):
 136         """Return True if the keyed message exists, False otherwise."""
 137         raise NotImplementedError('Method must be implemented by subclass')
 138
 139     def __contains__(self, key):
 140         return self.has_key(key)
 141
 142     def __len__(self):
 143         """Return a count of messages in the mailbox."""
 144         raise NotImplementedError('Method must be implemented by subclass')
 145
 146     def clear(self):
 147         """Delete all messages."""
 148         for key in self.iterkeys():
 149             self.discard(key)
 150
 151     def pop(self, key, default=None):
 152         """Delete the keyed message and return it, or default."""
 153         try:
 154             result = self[key]
 155         except KeyError:
 156             return default
 157         self.discard(key)
 158         return result
 159
 160     def popitem(self):
 161         """Delete an arbitrary (key, message) pair and return it."""
 162         for key in self.iterkeys():
 163             return (key, self.pop(key))     # This is only run once.
 164         else:
 165             raise KeyError('No messages in mailbox')
 166
 167     def update(self, arg=None):
 168         """Change the messages that correspond to certain keys."""
 169         if hasattr(arg, 'iteritems'):
 170             source = arg.iteritems()
 171         elif hasattr(arg, 'items'):
 172             source = arg.items()
 173         else:
 174             source = arg
 175         bad_key = False
 176         for key, message in source:
 177             try:
 178                 self[key] = message
 179             except KeyError:
 180                 bad_key = True
 181         if bad_key:
 182             raise KeyError('No message with key(s)')
 183
 184     def flush(self):
 185         """Write any pending changes to the disk."""
 186         raise NotImplementedError('Method must be implemented by subclass')
 187
 188     def lock(self):
 189         """Lock the mailbox."""
 190         raise NotImplementedError('Method must be implemented by subclass')
 191
 192     def unlock(self):
 193         """Unlock the mailbox if it is locked."""
 194         raise NotImplementedError('Method must be implemented by subclass')
 195
 196     def close(self):
 197         """Flush and close the mailbox."""
 198         raise NotImplementedError('Method must be implemented by subclass')
 199
 200     def _dump_message(self, message, target, mangle_from_=False):
 201         # Most files are opened in binary mode to allow predictable seeking.
 202         # To get native line endings on disk, the user-friendly \n line endings
 203         # used in strings and by email.Message are translated here.
 204         """Dump message contents to target file."""
 205         if isinstance(message, email.message.Message):
 206             buffer = StringIO.StringIO()
 207             gen = email.generator.Generator(buffer, mangle_from_, 0)
 208             gen.flatten(message)
 209             buffer.seek(0)
 210             target.write(buffer.read().replace('\n', os.linesep))
 211         elif isinstance(message, str):
 212             if mangle_from_:
 213                 message = message.replace('\nFrom ', '\n>From ')
 214             message = message.replace('\n', os.linesep)
 215             target.write(message)
 216         elif hasattr(message, 'read'):
 217             while True:
 218                 line = message.readline()
 219                 if line == '':
 220                     break
 221                 if mangle_from_ and line.startswith('From '):
 222                     line = '>From ' + line[5:]
 223                 line = line.replace('\n', os.linesep)
 224                 target.write(line)
 225         else:
 226             raise TypeError('Invalid message type: %s' % type(message))
 227
 228
 229 class Maildir(Mailbox):
 230     """A qmail-style Maildir mailbox."""
 231
 232     colon = ':'
 233
 234     def __init__(self, dirname, factory=rfc822.Message, create=True):
 235         """Initialize a Maildir instance."""
 236         Mailbox.__init__(self, dirname, factory, create)
 237         if not os.path.exists(self._path):
 238             if create:
 239                 os.mkdir(self._path, 0700)
 240                 os.mkdir(os.path.join(self._path, 'tmp'), 0700)
 241                 os.mkdir(os.path.join(self._path, 'new'), 0700)
 242                 os.mkdir(os.path.join(self._path, 'cur'), 0700)
 243             else:
 244                 raise NoSuchMailboxError(self._path)
 245         self._toc = {}
 246         self._last_read = None          # Records last time we read cur/new
 247         # NOTE: we manually invalidate _last_read each time we do any
 248         # modifications ourselves, otherwise we might get tripped up by
 249         # bogus mtime behaviour on some systems (see issue #6896).
 250
 251     def add(self, message):
 252         """Add message and return assigned key."""
 253         tmp_file = self._create_tmp()
 254         try:
 255             self._dump_message(message, tmp_file)
 256         finally:
 257             _sync_close(tmp_file)
 258         if isinstance(message, MaildirMessage):
 259             subdir = message.get_subdir()
 260             suffix = self.colon + message.get_info()
 261             if suffix == self.colon:
 262                 suffix = ''
 263         else:
 264             subdir = 'new'
 265             suffix = ''
 266         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 267         dest = os.path.join(self._path, subdir, uniq + suffix)
 268         try:
 269             if hasattr(os, 'link'):
 270                 os.link(tmp_file.name, dest)
 271                 os.remove(tmp_file.name)
 272             else:
 273                 os.rename(tmp_file.name, dest)
 274         except OSError, e:
 275             os.remove(tmp_file.name)
 276             if e.errno == errno.EEXIST:
 277                 raise ExternalClashError('Name clash with existing message: %s'
 278                                          % dest)
 279             else:
 280                 raise
 281         if isinstance(message, MaildirMessage):
 282             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 283         # Invalidate cached toc
 284         self._last_read = None
 285         return uniq
 286
 287     def remove(self, key):
 288         """Remove the keyed message; raise KeyError if it doesn't exist."""
 289         os.remove(os.path.join(self._path, self._lookup(key)))
 290         # Invalidate cached toc (only on success)
 291         self._last_read = None
 292
 293     def discard(self, key):
 294         """If the keyed message exists, remove it."""
 295         # This overrides an inapplicable implementation in the superclass.
 296         try:
 297             self.remove(key)
 298         except KeyError:
 299             pass
 300         except OSError, e:
 301             if e.errno != errno.ENOENT:
 302                 raise
 303
 304     def __setitem__(self, key, message):
 305         """Replace the keyed message; raise KeyError if it doesn't exist."""
 306         old_subpath = self._lookup(key)
 307         temp_key = self.add(message)
 308         temp_subpath = self._lookup(temp_key)
 309         if isinstance(message, MaildirMessage):
 310             # temp's subdir and suffix were specified by message.
 311             dominant_subpath = temp_subpath
 312         else:
 313             # temp's subdir and suffix were defaults from add().
 314             dominant_subpath = old_subpath
 315         subdir = os.path.dirname(dominant_subpath)
 316         if self.colon in dominant_subpath:
 317             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 318         else:
 319             suffix = ''
 320         self.discard(key)
 321         new_path = os.path.join(self._path, subdir, key + suffix)
 322         os.rename(os.path.join(self._path, temp_subpath), new_path)
 323         if isinstance(message, MaildirMessage):
 324             os.utime(new_path, (os.path.getatime(new_path),
 325                                 message.get_date()))
 326         # Invalidate cached toc
 327         self._last_read = None
 328
 329     def get_message(self, key):
 330         """Return a Message representation or raise a KeyError."""
 331         subpath = self._lookup(key)
 332         f = open(os.path.join(self._path, subpath), 'r')
 333         try:
 334             if self._factory:
 335                 msg = self._factory(f)
 336             else:
 337                 msg = MaildirMessage(f)
 338         finally:
 339             f.close()
 340         subdir, name = os.path.split(subpath)
 341         msg.set_subdir(subdir)
 342         if self.colon in name:
 343             msg.set_info(name.split(self.colon)[-1])
 344         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 345         return msg
 346
 347     def get_string(self, key):
 348         """Return a string representation or raise a KeyError."""
 349         f = open(os.path.join(self._path, self._lookup(key)), 'r')
 350         try:
 351             return f.read()
 352         finally:
 353             f.close()
 354
 355     def get_file(self, key):
 356         """Return a file-like representation or raise a KeyError."""
 357         f = open(os.path.join(self._path, self._lookup(key)), 'rb')
 358         return _ProxyFile(f)
 359
 360     def iterkeys(self):
 361         """Return an iterator over keys."""
 362         self._refresh()
 363         for key in self._toc:
 364             try:
 365                 self._lookup(key)
 366             except KeyError:
 367                 continue
 368             yield key
 369
 370     def has_key(self, key):
 371         """Return True if the keyed message exists, False otherwise."""
 372         self._refresh()
 373         return key in self._toc
 374
 375     def __len__(self):
 376         """Return a count of messages in the mailbox."""
 377         self._refresh()
 378         return len(self._toc)
 379
 380     def flush(self):
 381         """Write any pending changes to disk."""
 382         # Maildir changes are always written immediately, so there's nothing
 383         # to do except invalidate our cached toc.
 384         self._last_read = None
 385
 386     def lock(self):
 387         """Lock the mailbox."""
 388         return
 389
 390     def unlock(self):
 391         """Unlock the mailbox if it is locked."""
 392         return
 393
 394     def close(self):
 395         """Flush and close the mailbox."""
 396         return
 397
 398     def list_folders(self):
 399         """Return a list of folder names."""
 400         result = []
 401         for entry in os.listdir(self._path):
 402             if len(entry) > 1 and entry[0] == '.' and \
 403                os.path.isdir(os.path.join(self._path, entry)):
 404                 result.append(entry[1:])
 405         return result
 406
 407     def get_folder(self, folder):
 408         """Return a Maildir instance for the named folder."""
 409         return Maildir(os.path.join(self._path, '.' + folder),
 410                        factory=self._factory,
 411                        create=False)
 412
 413     def add_folder(self, folder):
 414         """Create a folder and return a Maildir instance representing it."""
 415         path = os.path.join(self._path, '.' + folder)
 416         result = Maildir(path, factory=self._factory)
 417         maildirfolder_path = os.path.join(path, 'maildirfolder')
 418         if not os.path.exists(maildirfolder_path):
 419             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
 420                 0666))
 421         return result
 422
 423     def remove_folder(self, folder):
 424         """Delete the named folder, which must be empty."""
 425         path = os.path.join(self._path, '.' + folder)
 426         for entry in os.listdir(os.path.join(path, 'new')) + \
 427                      os.listdir(os.path.join(path, 'cur')):
 428             if len(entry) < 1 or entry[0] != '.':
 429                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 430         for entry in os.listdir(path):
 431             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 432                os.path.isdir(os.path.join(path, entry)):
 433                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 434                                     (folder, entry))
 435         for root, dirs, files in os.walk(path, topdown=False):
 436             for entry in files:
 437                 os.remove(os.path.join(root, entry))
 438             for entry in dirs:
 439                 os.rmdir(os.path.join(root, entry))
 440         os.rmdir(path)
 441
 442     def clean(self):
 443         """Delete old files in "tmp"."""
 444         now = time.time()
 445         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 446             path = os.path.join(self._path, 'tmp', entry)
 447             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 448                 os.remove(path)
 449
 450     _count = 1  # This is used to generate unique file names.
 451
 452     def _create_tmp(self):
 453         """Create a file in the tmp subdirectory and open and return it."""
 454         now = time.time()
 455         hostname = socket.gethostname()
 456         if '/' in hostname:
 457             hostname = hostname.replace('/', r'\057')
 458         if ':' in hostname:
 459             hostname = hostname.replace(':', r'\072')
 460         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 461                                     Maildir._count, hostname)
 462         path = os.path.join(self._path, 'tmp', uniq)
 463         try:
 464             os.stat(path)
 465         except OSError, e:
 466             if e.errno == errno.ENOENT:
 467                 Maildir._count += 1
 468                 try:
 469                     return _create_carefully(path)
 470                 except OSError, e:
 471                     if e.errno != errno.EEXIST:
 472                         raise
 473             else:
 474                 raise
 475
 476         # Fall through to here if stat succeeded or open raised EEXIST.
 477         raise ExternalClashError('Name clash prevented file creation: %s' %
 478                                  path)
 479
 480     def _refresh(self):
 481         """Update table of contents mapping."""
 482         if self._last_read is not None:
 483             for subdir in ('new', 'cur'):
 484                 mtime = os.path.getmtime(os.path.join(self._path, subdir))
 485                 if mtime > self._last_read:
 486                     break
 487             else:
 488                 return
 489
 490         # We record the current time - 1sec so that, if _refresh() is called
 491         # again in the same second, we will always re-read the mailbox
 492         # just in case it's been modified.  (os.path.mtime() only has
 493         # 1sec resolution.)  This results in a few unnecessary re-reads
 494         # when _refresh() is called multiple times in the same second,
 495         # but once the clock ticks over, we will only re-read as needed.
 496         now = time.time() - 1
 497
 498         self._toc = {}
 499         def update_dir (subdir):
 500             path = os.path.join(self._path, subdir)
 501             for entry in os.listdir(path):
 502                 p = os.path.join(path, entry)
 503                 if os.path.isdir(p):
 504                     continue
 505                 uniq = entry.split(self.colon)[0]
 506                 self._toc[uniq] = os.path.join(subdir, entry)
 507
 508         update_dir('new')
 509         update_dir('cur')
 510
 511         self._last_read = now
 512
 513     def _lookup(self, key):
 514         """Use TOC to return subpath for given key, or raise a KeyError."""
 515         try:
 516             if os.path.exists(os.path.join(self._path, self._toc[key])):
 517                 return self._toc[key]
 518         except KeyError:
 519             pass
 520         self._refresh()
 521         try:
 522             return self._toc[key]
 523         except KeyError:
 524             raise KeyError('No message with key: %s' % key)
 525
 526     # This method is for backward compatibility only.
 527     def next(self):
 528         """Return the next message in a one-time iteration."""
 529         if not hasattr(self, '_onetime_keys'):
 530             self._onetime_keys = self.iterkeys()
 531         while True:
 532             try:
 533                 return self[self._onetime_keys.next()]
 534             except StopIteration:
 535                 return None
 536             except KeyError:
 537                 continue
 538
 539
 540 class _singlefileMailbox(Mailbox):
 541     """A single-file mailbox."""
 542
 543     def __init__(self, path, factory=None, create=True):
 544         """Initialize a single-file mailbox."""
 545         Mailbox.__init__(self, path, factory, create)
 546         try:
 547             f = open(self._path, 'rb+')
 548         except IOError, e:
 549             if e.errno == errno.ENOENT:
 550                 if create:
 551                     f = open(self._path, 'wb+')
 552                 else:
 553                     raise NoSuchMailboxError(self._path)
 554             elif e.errno == errno.EACCES:
 555                 f = open(self._path, 'rb')
 556             else:
 557                 raise
 558         self._file = f
 559         self._toc = None
 560         self._next_key = 0
 561         self._pending = False   # No changes require rewriting the file.
 562         self._locked = False
 563         self._file_length = None        # Used to record mailbox size
 564
 565     def add(self, message):
 566         """Add message and return assigned key."""
 567         self._lookup()
 568         self._toc[self._next_key] = self._append_message(message)
 569         self._next_key += 1
 570         self._pending = True
 571         return self._next_key - 1
 572
 573     def remove(self, key):
 574         """Remove the keyed message; raise KeyError if it doesn't exist."""
 575         self._lookup(key)
 576         del self._toc[key]
 577         self._pending = True
 578
 579     def __setitem__(self, key, message):
 580         """Replace the keyed message; raise KeyError if it doesn't exist."""
 581         self._lookup(key)
 582         self._toc[key] = self._append_message(message)
 583         self._pending = True
 584
 585     def iterkeys(self):
 586         """Return an iterator over keys."""
 587         self._lookup()
 588         for key in self._toc.keys():
 589             yield key
 590
 591     def has_key(self, key):
 592         """Return True if the keyed message exists, False otherwise."""
 593         self._lookup()
 594         return key in self._toc
 595
 596     def __len__(self):
 597         """Return a count of messages in the mailbox."""
 598         self._lookup()
 599         return len(self._toc)
 600
 601     def lock(self):
 602         """Lock the mailbox."""
 603         if not self._locked:
 604             _lock_file(self._file)
 605             self._locked = True
 606
 607     def unlock(self):
 608         """Unlock the mailbox if it is locked."""
 609         if self._locked:
 610             _unlock_file(self._file)
 611             self._locked = False
 612
 613     def flush(self):
 614         """Write any pending changes to disk."""
 615         if not self._pending:
 616             return
 617
 618         # In order to be writing anything out at all, self._toc must
 619         # already have been generated (and presumably has been modified
 620         # by adding or deleting an item).
 621         assert self._toc is not None
 622
 623         # Check length of self._file; if it's changed, some other process
 624         # has modified the mailbox since we scanned it.
 625         self._file.seek(0, 2)
 626         cur_len = self._file.tell()
 627         if cur_len != self._file_length:
 628             raise ExternalClashError('Size of mailbox file changed '
 629                                      '(expected %i, found %i)' %
 630                                      (self._file_length, cur_len))
 631
 632         new_file = _create_temporary(self._path)
 633         try:
 634             new_toc = {}
 635             self._pre_mailbox_hook(new_file)
 636             for key in sorted(self._toc.keys()):
 637                 start, stop = self._toc[key]
 638                 self._file.seek(start)
 639                 self._pre_message_hook(new_file)
 640                 new_start = new_file.tell()
 641                 while True:
 642                     buffer = self._file.read(min(4096,
 643                                                  stop - self._file.tell()))
 644                     if buffer == '':
 645                         break
 646                     new_file.write(buffer)
 647                 new_toc[key] = (new_start, new_file.tell())
 648                 self._post_message_hook(new_file)
 649         except:
 650             new_file.close()
 651             os.remove(new_file.name)
 652             raise
 653         _sync_close(new_file)
 654         # self._file is about to get replaced, so no need to sync.
 655         self._file.close()
 656         try:
 657             os.rename(new_file.name, self._path)
 658         except OSError, e:
 659             if e.errno == errno.EEXIST or \
 660               (os.name == 'os2' and e.errno == errno.EACCES):
 661                 os.remove(self._path)
 662                 os.rename(new_file.name, self._path)
 663             else:
 664                 raise
 665         self._file = open(self._path, 'rb+')
 666         self._toc = new_toc
 667         self._pending = False
 668         if self._locked:
 669             _lock_file(self._file, dotlock=False)
 670
 671     def _pre_mailbox_hook(self, f):
 672         """Called before writing the mailbox to file f."""
 673         return
 674
 675     def _pre_message_hook(self, f):
 676         """Called before writing each message to file f."""
 677         return
 678
 679     def _post_message_hook(self, f):
 680         """Called after writing each message to file f."""
 681         return
 682
 683     def close(self):
 684         """Flush and close the mailbox."""
 685         self.flush()
 686         if self._locked:
 687             self.unlock()
 688         self._file.close()  # Sync has been done by self.flush() above.
 689
 690     def _lookup(self, key=None):
 691         """Return (start, stop) or raise KeyError."""
 692         if self._toc is None:
 693             self._generate_toc()
 694         if key is not None:
 695             try:
 696                 return self._toc[key]
 697             except KeyError:
 698                 raise KeyError('No message with key: %s' % key)
 699
 700     def _append_message(self, message):
 701         """Append message to mailbox and return (start, stop) offsets."""
 702         self._file.seek(0, 2)
 703         self._pre_message_hook(self._file)
 704         offsets = self._install_message(message)
 705         self._post_message_hook(self._file)
 706         self._file.flush()
 707         self._file_length = self._file.tell()  # Record current length of mailbox
 708         return offsets
 709
 710
 711
 712 class _mboxMMDF(_singlefileMailbox):
 713     """An mbox or MMDF mailbox."""
 714
 715     _mangle_from_ = True
 716
 717     def get_message(self, key):
 718         """Return a Message representation or raise a KeyError."""
 719         start, stop = self._lookup(key)
 720         self._file.seek(start)
 721         from_line = self._file.readline().replace(os.linesep, '')
 722         string = self._file.read(stop - self._file.tell())
 723         msg = self._message_factory(string.replace(os.linesep, '\n'))
 724         msg.set_from(from_line[5:])
 725         return msg
 726
 727     def get_string(self, key, from_=False):
 728         """Return a string representation or raise a KeyError."""
 729         start, stop = self._lookup(key)
 730         self._file.seek(start)
 731         if not from_:
 732             self._file.readline()
 733         string = self._file.read(stop - self._file.tell())
 734         return string.replace(os.linesep, '\n')
 735
 736     def get_file(self, key, from_=False):
 737         """Return a file-like representation or raise a KeyError."""
 738         start, stop = self._lookup(key)
 739         self._file.seek(start)
 740         if not from_:
 741             self._file.readline()
 742         return _PartialFile(self._file, self._file.tell(), stop)
 743
 744     def _install_message(self, message):
 745         """Format a message and blindly write to self._file."""
 746         from_line = None
 747         if isinstance(message, str) and message.startswith('From '):
 748             newline = message.find('\n')
 749             if newline != -1:
 750                 from_line = message[:newline]
 751                 message = message[newline + 1:]
 752             else:
 753                 from_line = message
 754                 message = ''
 755         elif isinstance(message, _mboxMMDFMessage):
 756             from_line = 'From ' + message.get_from()
 757         elif isinstance(message, email.message.Message):
 758             from_line = message.get_unixfrom()  # May be None.
 759         if from_line is None:
 760             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 761         start = self._file.tell()
 762         self._file.write(from_line + os.linesep)
 763         self._dump_message(message, self._file, self._mangle_from_)
 764         stop = self._file.tell()
 765         return (start, stop)
 766
 767
 768 class mbox(_mboxMMDF):
 769     """A classic mbox mailbox."""
 770
 771     _mangle_from_ = True
 772
 773     def __init__(self, path, factory=None, create=True):
 774         """Initialize an mbox mailbox."""
 775         self._message_factory = mboxMessage
 776         _mboxMMDF.__init__(self, path, factory, create)
 777
 778     def _pre_message_hook(self, f):
 779         """Called before writing each message to file f."""
 780         if f.tell() != 0:
 781             f.write(os.linesep)
 782
 783     def _generate_toc(self):
 784         """Generate key-to-(start, stop) table of contents."""
 785         starts, stops = [], []
 786         self._file.seek(0)
 787         while True:
 788             line_pos = self._file.tell()
 789             line = self._file.readline()
 790             if line.startswith('From '):
 791                 if len(stops) < len(starts):
 792                     stops.append(line_pos - len(os.linesep))
 793                 starts.append(line_pos)
 794             elif line == '':
 795                 stops.append(line_pos)
 796                 break
 797         self._toc = dict(enumerate(zip(starts, stops)))
 798         self._next_key = len(self._toc)
 799         self._file_length = self._file.tell()
 800
 801
 802 class MMDF(_mboxMMDF):
 803     """An MMDF mailbox."""
 804
 805     def __init__(self, path, factory=None, create=True):
 806         """Initialize an MMDF mailbox."""
 807         self._message_factory = MMDFMessage
 808         _mboxMMDF.__init__(self, path, factory, create)
 809
 810     def _pre_message_hook(self, f):
 811         """Called before writing each message to file f."""
 812         f.write('\001\001\001\001' + os.linesep)
 813
 814     def _post_message_hook(self, f):
 815         """Called after writing each message to file f."""
 816         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 817
 818     def _generate_toc(self):
 819         """Generate key-to-(start, stop) table of contents."""
 820         starts, stops = [], []
 821         self._file.seek(0)
 822         next_pos = 0
 823         while True:
 824             line_pos = next_pos
 825             line = self._file.readline()
 826             next_pos = self._file.tell()
 827             if line.startswith('\001\001\001\001' + os.linesep):
 828                 starts.append(next_pos)
 829                 while True:
 830                     line_pos = next_pos
 831                     line = self._file.readline()
 832                     next_pos = self._file.tell()
 833                     if line == '\001\001\001\001' + os.linesep:
 834                         stops.append(line_pos - len(os.linesep))
 835                         break
 836                     elif line == '':
 837                         stops.append(line_pos)
 838                         break
 839             elif line == '':
 840                 break
 841         self._toc = dict(enumerate(zip(starts, stops)))
 842         self._next_key = len(self._toc)
 843         self._file.seek(0, 2)
 844         self._file_length = self._file.tell()
 845
 846
 847 class MH(Mailbox):
 848     """An MH mailbox."""
 849
 850     def __init__(self, path, factory=None, create=True):
 851         """Initialize an MH instance."""
 852         Mailbox.__init__(self, path, factory, create)
 853         if not os.path.exists(self._path):
 854             if create:
 855                 os.mkdir(self._path, 0700)
 856                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 857                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
 858             else:
 859                 raise NoSuchMailboxError(self._path)
 860         self._locked = False
 861
 862     def add(self, message):
 863         """Add message and return assigned key."""
 864         keys = self.keys()
 865         if len(keys) == 0:
 866             new_key = 1
 867         else:
 868             new_key = max(keys) + 1
 869         new_path = os.path.join(self._path, str(new_key))
 870         f = _create_carefully(new_path)
 871         try:
 872             if self._locked:
 873                 _lock_file(f)
 874             try:
 875                 self._dump_message(message, f)
 876                 if isinstance(message, MHMessage):
 877                     self._dump_sequences(message, new_key)
 878             finally:
 879                 if self._locked:
 880                     _unlock_file(f)
 881         finally:
 882             _sync_close(f)
 883         return new_key
 884
 885     def remove(self, key):
 886         """Remove the keyed message; raise KeyError if it doesn't exist."""
 887         path = os.path.join(self._path, str(key))
 888         try:
 889             f = open(path, 'rb+')
 890         except IOError, e:
 891             if e.errno == errno.ENOENT:
 892                 raise KeyError('No message with key: %s' % key)
 893             else:
 894                 raise
 895         else:
 896             f.close()
 897             os.remove(path)
 898
 899     def __setitem__(self, key, message):
 900         """Replace the keyed message; raise KeyError if it doesn't exist."""
 901         path = os.path.join(self._path, str(key))
 902         try:
 903             f = open(path, 'rb+')
 904         except IOError, e:
 905             if e.errno == errno.ENOENT:
 906                 raise KeyError('No message with key: %s' % key)
 907             else:
 908                 raise
 909         try:
 910             if self._locked:
 911                 _lock_file(f)
 912             try:
 913                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 914                 self._dump_message(message, f)
 915                 if isinstance(message, MHMessage):
 916                     self._dump_sequences(message, key)
 917             finally:
 918                 if self._locked:
 919                     _unlock_file(f)
 920         finally:
 921             _sync_close(f)
 922
 923     def get_message(self, key):
 924         """Return a Message representation or raise a KeyError."""
 925         try:
 926             if self._locked:
 927                 f = open(os.path.join(self._path, str(key)), 'r+')
 928             else:
 929                 f = open(os.path.join(self._path, str(key)), 'r')
 930         except IOError, e:
 931             if e.errno == errno.ENOENT:
 932                 raise KeyError('No message with key: %s' % key)
 933             else:
 934                 raise
 935         try:
 936             if self._locked:
 937                 _lock_file(f)
 938             try:
 939                 msg = MHMessage(f)
 940             finally:
 941                 if self._locked:
 942                     _unlock_file(f)
 943         finally:
 944             f.close()
 945         for name, key_list in self.get_sequences().iteritems():
 946             if key in key_list:
 947                 msg.add_sequence(name)
 948         return msg
 949
 950     def get_string(self, key):
 951         """Return a string representation or raise a KeyError."""
 952         try:
 953             if self._locked:
 954                 f = open(os.path.join(self._path, str(key)), 'r+')
 955             else:
 956                 f = open(os.path.join(self._path, str(key)), 'r')
 957         except IOError, e:
 958             if e.errno == errno.ENOENT:
 959                 raise KeyError('No message with key: %s' % key)
 960             else:
 961                 raise
 962         try:
 963             if self._locked:
 964                 _lock_file(f)
 965             try:
 966                 return f.read()
 967             finally:
 968                 if self._locked:
 969                     _unlock_file(f)
 970         finally:
 971             f.close()
 972
 973     def get_file(self, key):
 974         """Return a file-like representation or raise a KeyError."""
 975         try:
 976             f = open(os.path.join(self._path, str(key)), 'rb')
 977         except IOError, e:
 978             if e.errno == errno.ENOENT:
 979                 raise KeyError('No message with key: %s' % key)
 980             else:
 981                 raise
 982         return _ProxyFile(f)
 983
 984     def iterkeys(self):
 985         """Return an iterator over keys."""
 986         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 987                                       if entry.isdigit()))
 988
 989     def has_key(self, key):
 990         """Return True if the keyed message exists, False otherwise."""
 991         return os.path.exists(os.path.join(self._path, str(key)))
 992
 993     def __len__(self):
 994         """Return a count of messages in the mailbox."""
 995         return len(list(self.iterkeys()))
 996
 997     def lock(self):
 998         """Lock the mailbox."""
 999         if not self._locked:
1000             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
1001             _lock_file(self._file)
1002             self._locked = True
1003
1004     def unlock(self):
1005         """Unlock the mailbox if it is locked."""
1006         if self._locked:
1007             _unlock_file(self._file)
1008             _sync_close(self._file)
1009             del self._file
1010             self._locked = False
1011
1012     def flush(self):
1013         """Write any pending changes to the disk."""
1014         return
1015
1016     def close(self):
1017         """Flush and close the mailbox."""
1018         if self._locked:
1019             self.unlock()
1020
1021     def list_folders(self):
1022         """Return a list of folder names."""
1023         result = []
1024         for entry in os.listdir(self._path):
1025             if os.path.isdir(os.path.join(self._path, entry)):
1026                 result.append(entry)
1027         return result
1028
1029     def get_folder(self, folder):
1030         """Return an MH instance for the named folder."""
1031         return MH(os.path.join(self._path, folder),
1032                   factory=self._factory, create=False)
1033
1034     def add_folder(self, folder):
1035         """Create a folder and return an MH instance representing it."""
1036         return MH(os.path.join(self._path, folder),
1037                   factory=self._factory)
1038
1039     def remove_folder(self, folder):
1040         """Delete the named folder, which must be empty."""
1041         path = os.path.join(self._path, folder)
1042         entries = os.listdir(path)
1043         if entries == ['.mh_sequences']:
1044             os.remove(os.path.join(path, '.mh_sequences'))
1045         elif entries == []:
1046             pass
1047         else:
1048             raise NotEmptyError('Folder not empty: %s' % self._path)
1049         os.rmdir(path)
1050
1051     def get_sequences(self):
1052         """Return a name-to-key-list dictionary to define each sequence."""
1053         results = {}
1054         f = open(os.path.join(self._path, '.mh_sequences'), 'r')
1055         try:
1056             all_keys = set(self.keys())
1057             for line in f:
1058                 try:
1059                     name, contents = line.split(':')
1060                     keys = set()
1061                     for spec in contents.split():
1062                         if spec.isdigit():
1063                             keys.add(int(spec))
1064                         else:
1065                             start, stop = (int(x) for x in spec.split('-'))
1066                             keys.update(range(start, stop + 1))
1067                     results[name] = [key for key in sorted(keys) \
1068                                          if key in all_keys]
1069                     if len(results[name]) == 0:
1070                         del results[name]
1071                 except ValueError:
1072                     raise FormatError('Invalid sequence specification: %s' %
1073                                       line.rstrip())
1074         finally:
1075             f.close()
1076         return results
1077
1078     def set_sequences(self, sequences):
1079         """Set sequences using the given name-to-key-list dictionary."""
1080         f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
1081         try:
1082             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1083             for name, keys in sequences.iteritems():
1084                 if len(keys) == 0:
1085                     continue
1086                 f.write('%s:' % name)
1087                 prev = None
1088                 completing = False
1089                 for key in sorted(set(keys)):
1090                     if key - 1 == prev:
1091                         if not completing:
1092                             completing = True
1093                             f.write('-')
1094                     elif completing:
1095                         completing = False
1096                         f.write('%s %s' % (prev, key))
1097                     else:
1098                         f.write(' %s' % key)
1099                     prev = key
1100                 if completing:
1101                     f.write(str(prev) + '\n')
1102                 else:
1103                     f.write('\n')
1104         finally:
1105             _sync_close(f)
1106
1107     def pack(self):
1108         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1109         sequences = self.get_sequences()
1110         prev = 0
1111         changes = []
1112         for key in self.iterkeys():
1113             if key - 1 != prev:
1114                 changes.append((key, prev + 1))
1115                 if hasattr(os, 'link'):
1116                     os.link(os.path.join(self._path, str(key)),
1117                             os.path.join(self._path, str(prev + 1)))
1118                     os.unlink(os.path.join(self._path, str(key)))
1119                 else:
1120                     os.rename(os.path.join(self._path, str(key)),
1121                               os.path.join(self._path, str(prev + 1)))
1122             prev += 1
1123         self._next_key = prev + 1
1124         if len(changes) == 0:
1125             return
1126         for name, key_list in sequences.items():
1127             for old, new in changes:
1128                 if old in key_list:
1129                     key_list[key_list.index(old)] = new
1130         self.set_sequences(sequences)
1131
1132     def _dump_sequences(self, message, key):
1133         """Inspect a new MHMessage and update sequences appropriately."""
1134         pending_sequences = message.get_sequences()
1135         all_sequences = self.get_sequences()
1136         for name, key_list in all_sequences.iteritems():
1137             if name in pending_sequences:
1138                 key_list.append(key)
1139             elif key in key_list:
1140                 del key_list[key_list.index(key)]
1141         for sequence in pending_sequences:
1142             if sequence not in all_sequences:
1143                 all_sequences[sequence] = [key]
1144         self.set_sequences(all_sequences)
1145
1146
1147 class Babyl(_singlefileMailbox):
1148     """An Rmail-style Babyl mailbox."""
1149
1150     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1151                                  'forwarded', 'edited', 'resent'))
1152
1153     def __init__(self, path, factory=None, create=True):
1154         """Initialize a Babyl mailbox."""
1155         _singlefileMailbox.__init__(self, path, factory, create)
1156         self._labels = {}
1157
1158     def add(self, message):
1159         """Add message and return assigned key."""
1160         key = _singlefileMailbox.add(self, message)
1161         if isinstance(message, BabylMessage):
1162             self._labels[key] = message.get_labels()
1163         return key
1164
1165     def remove(self, key):
1166         """Remove the keyed message; raise KeyError if it doesn't exist."""
1167         _singlefileMailbox.remove(self, key)
1168         if key in self._labels:
1169             del self._labels[key]
1170
1171     def __setitem__(self, key, message):
1172         """Replace the keyed message; raise KeyError if it doesn't exist."""
1173         _singlefileMailbox.__setitem__(self, key, message)
1174         if isinstance(message, BabylMessage):
1175             self._labels[key] = message.get_labels()
1176
1177     def get_message(self, key):
1178         """Return a Message representation or raise a KeyError."""
1179         start, stop = self._lookup(key)
1180         self._file.seek(start)
1181         self._file.readline()   # Skip '1,' line specifying labels.
1182         original_headers = StringIO.StringIO()
1183         while True:
1184             line = self._file.readline()
1185             if line == '*** EOOH ***' + os.linesep or line == '':
1186                 break
1187             original_headers.write(line.replace(os.linesep, '\n'))
1188         visible_headers = StringIO.StringIO()
1189         while True:
1190             line = self._file.readline()
1191             if line == os.linesep or line == '':
1192                 break
1193             visible_headers.write(line.replace(os.linesep, '\n'))
1194         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1195                                                                  '\n')
1196         msg = BabylMessage(original_headers.getvalue() + body)
1197         msg.set_visible(visible_headers.getvalue())
1198         if key in self._labels:
1199             msg.set_labels(self._labels[key])
1200         return msg
1201
1202     def get_string(self, key):
1203         """Return a string representation or raise a KeyError."""
1204         start, stop = self._lookup(key)
1205         self._file.seek(start)
1206         self._file.readline()   # Skip '1,' line specifying labels.
1207         original_headers = StringIO.StringIO()
1208         while True:
1209             line = self._file.readline()
1210             if line == '*** EOOH ***' + os.linesep or line == '':
1211                 break
1212             original_headers.write(line.replace(os.linesep, '\n'))
1213         while True:
1214             line = self._file.readline()
1215             if line == os.linesep or line == '':
1216                 break
1217         return original_headers.getvalue() + \
1218                self._file.read(stop - self._file.tell()).replace(os.linesep,
1219                                                                  '\n')
1220
1221     def get_file(self, key):
1222         """Return a file-like representation or raise a KeyError."""
1223         return StringIO.StringIO(self.get_string(key).replace('\n',
1224                                                               os.linesep))
1225
1226     def get_labels(self):
1227         """Return a list of user-defined labels in the mailbox."""
1228         self._lookup()
1229         labels = set()
1230         for label_list in self._labels.values():
1231             labels.update(label_list)
1232         labels.difference_update(self._special_labels)
1233         return list(labels)
1234
1235     def _generate_toc(self):
1236         """Generate key-to-(start, stop) table of contents."""
1237         starts, stops = [], []
1238         self._file.seek(0)
1239         next_pos = 0
1240         label_lists = []
1241         while True:
1242             line_pos = next_pos
1243             line = self._file.readline()
1244             next_pos = self._file.tell()
1245             if line == '\037\014' + os.linesep:
1246                 if len(stops) < len(starts):
1247                     stops.append(line_pos - len(os.linesep))
1248                 starts.append(next_pos)
1249                 labels = [label.strip() for label
1250                                         in self._file.readline()[1:].split(',')
1251                                         if label.strip() != '']
1252                 label_lists.append(labels)
1253             elif line == '\037' or line == '\037' + os.linesep:
1254                 if len(stops) < len(starts):
1255                     stops.append(line_pos - len(os.linesep))
1256             elif line == '':
1257                 stops.append(line_pos - len(os.linesep))
1258                 break
1259         self._toc = dict(enumerate(zip(starts, stops)))
1260         self._labels = dict(enumerate(label_lists))
1261         self._next_key = len(self._toc)
1262         self._file.seek(0, 2)
1263         self._file_length = self._file.tell()
1264
1265     def _pre_mailbox_hook(self, f):
1266         """Called before writing the mailbox to file f."""
1267         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1268                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1269                  os.linesep))
1270
1271     def _pre_message_hook(self, f):
1272         """Called before writing each message to file f."""
1273         f.write('\014' + os.linesep)
1274
1275     def _post_message_hook(self, f):
1276         """Called after writing each message to file f."""
1277         f.write(os.linesep + '\037')
1278
1279     def _install_message(self, message):
1280         """Write message contents and return (start, stop)."""
1281         start = self._file.tell()
1282         if isinstance(message, BabylMessage):
1283             special_labels = []
1284             labels = []
1285             for label in message.get_labels():
1286                 if label in self._special_labels:
1287                     special_labels.append(label)
1288                 else:
1289                     labels.append(label)
1290             self._file.write('1')
1291             for label in special_labels:
1292                 self._file.write(', ' + label)
1293             self._file.write(',,')
1294             for label in labels:
1295                 self._file.write(' ' + label + ',')
1296             self._file.write(os.linesep)
1297         else:
1298             self._file.write('1,,' + os.linesep)
1299         if isinstance(message, email.message.Message):
1300             orig_buffer = StringIO.StringIO()
1301             orig_generator = email.generator.Generator(orig_buffer, False, 0)
1302             orig_generator.flatten(message)
1303             orig_buffer.seek(0)
1304             while True:
1305                 line = orig_buffer.readline()
1306                 self._file.write(line.replace('\n', os.linesep))
1307                 if line == '\n' or line == '':
1308                     break
1309             self._file.write('*** EOOH ***' + os.linesep)
1310             if isinstance(message, BabylMessage):
1311                 vis_buffer = StringIO.StringIO()
1312                 vis_generator = email.generator.Generator(vis_buffer, False, 0)
1313                 vis_generator.flatten(message.get_visible())
1314                 while True:
1315                     line = vis_buffer.readline()
1316                     self._file.write(line.replace('\n', os.linesep))
1317                     if line == '\n' or line == '':
1318                         break
1319             else:
1320                 orig_buffer.seek(0)
1321                 while True:
1322                     line = orig_buffer.readline()
1323                     self._file.write(line.replace('\n', os.linesep))
1324                     if line == '\n' or line == '':
1325                         break
1326             while True:
1327                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1328                 if buffer == '':
1329                     break
1330                 self._file.write(buffer.replace('\n', os.linesep))
1331         elif isinstance(message, str):
1332             body_start = message.find('\n\n') + 2
1333             if body_start - 2 != -1:
1334                 self._file.write(message[:body_start].replace('\n',
1335                                                               os.linesep))
1336                 self._file.write('*** EOOH ***' + os.linesep)
1337                 self._file.write(message[:body_start].replace('\n',
1338                                                               os.linesep))
1339                 self._file.write(message[body_start:].replace('\n',
1340                                                               os.linesep))
1341             else:
1342                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1343                 self._file.write(message.replace('\n', os.linesep))
1344         elif hasattr(message, 'readline'):
1345             original_pos = message.tell()
1346             first_pass = True
1347             while True:
1348                 line = message.readline()
1349                 self._file.write(line.replace('\n', os.linesep))
1350                 if line == '\n' or line == '':
1351                     self._file.write('*** EOOH ***' + os.linesep)
1352                     if first_pass:
1353                         first_pass = False
1354                         message.seek(original_pos)
1355                     else:
1356                         break
1357             while True:
1358                 buffer = message.read(4096)     # Buffer size is arbitrary.
1359                 if buffer == '':
1360                     break
1361                 self._file.write(buffer.replace('\n', os.linesep))
1362         else:
1363             raise TypeError('Invalid message type: %s' % type(message))
1364         stop = self._file.tell()
1365         return (start, stop)
1366
1367
1368 class Message(email.message.Message):
1369     """Message with mailbox-format-specific properties."""
1370
1371     def __init__(self, message=None):
1372         """Initialize a Message instance."""
1373         if isinstance(message, email.message.Message):
1374             self._become_message(copy.deepcopy(message))
1375             if isinstance(message, Message):
1376                 message._explain_to(self)
1377         elif isinstance(message, str):
1378             self._become_message(email.message_from_string(message))
1379         elif hasattr(message, "read"):
1380             self._become_message(email.message_from_file(message))
1381         elif message is None:
1382             email.message.Message.__init__(self)
1383         else:
1384             raise TypeError('Invalid message type: %s' % type(message))
1385
1386     def _become_message(self, message):
1387         """Assume the non-format-specific state of message."""
1388         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1389                      'preamble', 'epilogue', 'defects', '_default_type'):
1390             self.__dict__[name] = message.__dict__[name]
1391
1392     def _explain_to(self, message):
1393         """Copy format-specific state to message insofar as possible."""
1394         if isinstance(message, Message):
1395             return  # There's nothing format-specific to explain.
1396         else:
1397             raise TypeError('Cannot convert to specified type')
1398
1399
1400 class MaildirMessage(Message):
1401     """Message with Maildir-specific properties."""
1402
1403     def __init__(self, message=None):
1404         """Initialize a MaildirMessage instance."""
1405         self._subdir = 'new'
1406         self._info = ''
1407         self._date = time.time()
1408         Message.__init__(self, message)
1409
1410     def get_subdir(self):
1411         """Return 'new' or 'cur'."""
1412         return self._subdir
1413
1414     def set_subdir(self, subdir):
1415         """Set subdir to 'new' or 'cur'."""
1416         if subdir == 'new' or subdir == 'cur':
1417             self._subdir = subdir
1418         else:
1419             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1420
1421     def get_flags(self):
1422         """Return as a string the flags that are set."""
1423         if self._info.startswith('2,'):
1424             return self._info[2:]
1425         else:
1426             return ''
1427
1428     def set_flags(self, flags):
1429         """Set the given flags and unset all others."""
1430         self._info = '2,' + ''.join(sorted(flags))
1431
1432     def add_flag(self, flag):
1433         """Set the given flag(s) without changing others."""
1434         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1435
1436     def remove_flag(self, flag):
1437         """Unset the given string flag(s) without changing others."""
1438         if self.get_flags() != '':
1439             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1440
1441     def get_date(self):
1442         """Return delivery date of message, in seconds since the epoch."""
1443         return self._date
1444
1445     def set_date(self, date):
1446         """Set delivery date of message, in seconds since the epoch."""
1447         try:
1448             self._date = float(date)
1449         except ValueError:
1450             raise TypeError("can't convert to float: %s" % date)
1451
1452     def get_info(self):
1453         """Get the message's "info" as a string."""
1454         return self._info
1455
1456     def set_info(self, info):
1457         """Set the message's "info" string."""
1458         if isinstance(info, str):
1459             self._info = info
1460         else:
1461             raise TypeError('info must be a string: %s' % type(info))
1462
1463     def _explain_to(self, message):
1464         """Copy Maildir-specific state to message insofar as possible."""
1465         if isinstance(message, MaildirMessage):
1466             message.set_flags(self.get_flags())
1467             message.set_subdir(self.get_subdir())
1468             message.set_date(self.get_date())
1469         elif isinstance(message, _mboxMMDFMessage):
1470             flags = set(self.get_flags())
1471             if 'S' in flags:
1472                 message.add_flag('R')
1473             if self.get_subdir() == 'cur':
1474                 message.add_flag('O')
1475             if 'T' in flags:
1476                 message.add_flag('D')
1477             if 'F' in flags:
1478                 message.add_flag('F')
1479             if 'R' in flags:
1480                 message.add_flag('A')
1481             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1482         elif isinstance(message, MHMessage):
1483             flags = set(self.get_flags())
1484             if 'S' not in flags:
1485                 message.add_sequence('unseen')
1486             if 'R' in flags:
1487                 message.add_sequence('replied')
1488             if 'F' in flags:
1489                 message.add_sequence('flagged')
1490         elif isinstance(message, BabylMessage):
1491             flags = set(self.get_flags())
1492             if 'S' not in flags:
1493                 message.add_label('unseen')
1494             if 'T' in flags:
1495                 message.add_label('deleted')
1496             if 'R' in flags:
1497                 message.add_label('answered')
1498             if 'P' in flags:
1499                 message.add_label('forwarded')
1500         elif isinstance(message, Message):
1501             pass
1502         else:
1503             raise TypeError('Cannot convert to specified type: %s' %
1504                             type(message))
1505
1506
1507 class _mboxMMDFMessage(Message):
1508     """Message with mbox- or MMDF-specific properties."""
1509
1510     def __init__(self, message=None):
1511         """Initialize an mboxMMDFMessage instance."""
1512         self.set_from('MAILER-DAEMON', True)
1513         if isinstance(message, email.message.Message):
1514             unixfrom = message.get_unixfrom()
1515             if unixfrom is not None and unixfrom.startswith('From '):
1516                 self.set_from(unixfrom[5:])
1517         Message.__init__(self, message)
1518
1519     def get_from(self):
1520         """Return contents of "From " line."""
1521         return self._from
1522
1523     def set_from(self, from_, time_=None):
1524         """Set "From " line, formatting and appending time_ if specified."""
1525         if time_ is not None:
1526             if time_ is True:
1527                 time_ = time.gmtime()
1528             from_ += ' ' + time.asctime(time_)
1529         self._from = from_
1530
1531     def get_flags(self):
1532         """Return as a string the flags that are set."""
1533         return self.get('Status', '') + self.get('X-Status', '')
1534
1535     def set_flags(self, flags):
1536         """Set the given flags and unset all others."""
1537         flags = set(flags)
1538         status_flags, xstatus_flags = '', ''
1539         for flag in ('R', 'O'):
1540             if flag in flags:
1541                 status_flags += flag
1542                 flags.remove(flag)
1543         for flag in ('D', 'F', 'A'):
1544             if flag in flags:
1545                 xstatus_flags += flag
1546                 flags.remove(flag)
1547         xstatus_flags += ''.join(sorted(flags))
1548         try:
1549             self.replace_header('Status', status_flags)
1550         except KeyError:
1551             self.add_header('Status', status_flags)
1552         try:
1553             self.replace_header('X-Status', xstatus_flags)
1554         except KeyError:
1555             self.add_header('X-Status', xstatus_flags)
1556
1557     def add_flag(self, flag):
1558         """Set the given flag(s) without changing others."""
1559         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1560
1561     def remove_flag(self, flag):
1562         """Unset the given string flag(s) without changing others."""
1563         if 'Status' in self or 'X-Status' in self:
1564             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1565
1566     def _explain_to(self, message):
1567         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1568         if isinstance(message, MaildirMessage):
1569             flags = set(self.get_flags())
1570             if 'O' in flags:
1571                 message.set_subdir('cur')
1572             if 'F' in flags:
1573                 message.add_flag('F')
1574             if 'A' in flags:
1575                 message.add_flag('R')
1576             if 'R' in flags:
1577                 message.add_flag('S')
1578             if 'D' in flags:
1579                 message.add_flag('T')
1580             del message['status']
1581             del message['x-status']
1582             maybe_date = ' '.join(self.get_from().split()[-5:])
1583             try:
1584                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1585                                                       '%a %b %d %H:%M:%S %Y')))
1586             except (ValueError, OverflowError):
1587                 pass
1588         elif isinstance(message, _mboxMMDFMessage):
1589             message.set_flags(self.get_flags())
1590             message.set_from(self.get_from())
1591         elif isinstance(message, MHMessage):
1592             flags = set(self.get_flags())
1593             if 'R' not in flags:
1594                 message.add_sequence('unseen')
1595             if 'A' in flags:
1596                 message.add_sequence('replied')
1597             if 'F' in flags:
1598                 message.add_sequence('flagged')
1599             del message['status']
1600             del message['x-status']
1601         elif isinstance(message, BabylMessage):
1602             flags = set(self.get_flags())
1603             if 'R' not in flags:
1604                 message.add_label('unseen')
1605             if 'D' in flags:
1606                 message.add_label('deleted')
1607             if 'A' in flags:
1608                 message.add_label('answered')
1609             del message['status']
1610             del message['x-status']
1611         elif isinstance(message, Message):
1612             pass
1613         else:
1614             raise TypeError('Cannot convert to specified type: %s' %
1615                             type(message))
1616
1617
1618 class mboxMessage(_mboxMMDFMessage):
1619     """Message with mbox-specific properties."""
1620
1621
1622 class MHMessage(Message):
1623     """Message with MH-specific properties."""
1624
1625     def __init__(self, message=None):
1626         """Initialize an MHMessage instance."""
1627         self._sequences = []
1628         Message.__init__(self, message)
1629
1630     def get_sequences(self):
1631         """Return a list of sequences that include the message."""
1632         return self._sequences[:]
1633
1634     def set_sequences(self, sequences):
1635         """Set the list of sequences that include the message."""
1636         self._sequences = list(sequences)
1637
1638     def add_sequence(self, sequence):
1639         """Add sequence to list of sequences including the message."""
1640         if isinstance(sequence, str):
1641             if not sequence in self._sequences:
1642                 self._sequences.append(sequence)
1643         else:
1644             raise TypeError('sequence must be a string: %s' % type(sequence))
1645
1646     def remove_sequence(self, sequence):
1647         """Remove sequence from the list of sequences including the message."""
1648         try:
1649             self._sequences.remove(sequence)
1650         except ValueError:
1651             pass
1652
1653     def _explain_to(self, message):
1654         """Copy MH-specific state to message insofar as possible."""
1655         if isinstance(message, MaildirMessage):
1656             sequences = set(self.get_sequences())
1657             if 'unseen' in sequences:
1658                 message.set_subdir('cur')
1659             else:
1660                 message.set_subdir('cur')
1661                 message.add_flag('S')
1662             if 'flagged' in sequences:
1663                 message.add_flag('F')
1664             if 'replied' in sequences:
1665                 message.add_flag('R')
1666         elif isinstance(message, _mboxMMDFMessage):
1667             sequences = set(self.get_sequences())
1668             if 'unseen' not in sequences:
1669                 message.add_flag('RO')
1670             else:
1671                 message.add_flag('O')
1672             if 'flagged' in sequences:
1673                 message.add_flag('F')
1674             if 'replied' in sequences:
1675                 message.add_flag('A')
1676         elif isinstance(message, MHMessage):
1677             for sequence in self.get_sequences():
1678                 message.add_sequence(sequence)
1679         elif isinstance(message, BabylMessage):
1680             sequences = set(self.get_sequences())
1681             if 'unseen' in sequences:
1682                 message.add_label('unseen')
1683             if 'replied' in sequences:
1684                 message.add_label('answered')
1685         elif isinstance(message, Message):
1686             pass
1687         else:
1688             raise TypeError('Cannot convert to specified type: %s' %
1689                             type(message))
1690
1691
1692 class BabylMessage(Message):
1693     """Message with Babyl-specific properties."""
1694
1695     def __init__(self, message=None):
1696         """Initialize an BabylMessage instance."""
1697         self._labels = []
1698         self._visible = Message()
1699         Message.__init__(self, message)
1700
1701     def get_labels(self):
1702         """Return a list of labels on the message."""
1703         return self._labels[:]
1704
1705     def set_labels(self, labels):
1706         """Set the list of labels on the message."""
1707         self._labels = list(labels)
1708
1709     def add_label(self, label):
1710         """Add label to list of labels on the message."""
1711         if isinstance(label, str):
1712             if label not in self._labels:
1713                 self._labels.append(label)
1714         else:
1715             raise TypeError('label must be a string: %s' % type(label))
1716
1717     def remove_label(self, label):
1718         """Remove label from the list of labels on the message."""
1719         try:
1720             self._labels.remove(label)
1721         except ValueError:
1722             pass
1723
1724     def get_visible(self):
1725         """Return a Message representation of visible headers."""
1726         return Message(self._visible)
1727
1728     def set_visible(self, visible):
1729         """Set the Message representation of visible headers."""
1730         self._visible = Message(visible)
1731
1732     def update_visible(self):
1733         """Update and/or sensibly generate a set of visible headers."""
1734         for header in self._visible.keys():
1735             if header in self:
1736                 self._visible.replace_header(header, self[header])
1737             else:
1738                 del self._visible[header]
1739         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1740             if header in self and header not in self._visible:
1741                 self._visible[header] = self[header]
1742
1743     def _explain_to(self, message):
1744         """Copy Babyl-specific state to message insofar as possible."""
1745         if isinstance(message, MaildirMessage):
1746             labels = set(self.get_labels())
1747             if 'unseen' in labels:
1748                 message.set_subdir('cur')
1749             else:
1750                 message.set_subdir('cur')
1751                 message.add_flag('S')
1752             if 'forwarded' in labels or 'resent' in labels:
1753                 message.add_flag('P')
1754             if 'answered' in labels:
1755                 message.add_flag('R')
1756             if 'deleted' in labels:
1757                 message.add_flag('T')
1758         elif isinstance(message, _mboxMMDFMessage):
1759             labels = set(self.get_labels())
1760             if 'unseen' not in labels:
1761                 message.add_flag('RO')
1762             else:
1763                 message.add_flag('O')
1764             if 'deleted' in labels:
1765                 message.add_flag('D')
1766             if 'answered' in labels:
1767                 message.add_flag('A')
1768         elif isinstance(message, MHMessage):
1769             labels = set(self.get_labels())
1770             if 'unseen' in labels:
1771                 message.add_sequence('unseen')
1772             if 'answered' in labels:
1773                 message.add_sequence('replied')
1774         elif isinstance(message, BabylMessage):
1775             message.set_visible(self.get_visible())
1776             for label in self.get_labels():
1777                 message.add_label(label)
1778         elif isinstance(message, Message):
1779             pass
1780         else:
1781             raise TypeError('Cannot convert to specified type: %s' %
1782                             type(message))
1783
1784
1785 class MMDFMessage(_mboxMMDFMessage):
1786     """Message with MMDF-specific properties."""
1787
1788
1789 class _ProxyFile:
1790     """A read-only wrapper of a file."""
1791
1792     def __init__(self, f, pos=None):
1793         """Initialize a _ProxyFile."""
1794         self._file = f
1795         if pos is None:
1796             self._pos = f.tell()
1797         else:
1798             self._pos = pos
1799
1800     def read(self, size=None):
1801         """Read bytes."""
1802         return self._read(size, self._file.read)
1803
1804     def readline(self, size=None):
1805         """Read a line."""
1806         return self._read(size, self._file.readline)
1807
1808     def readlines(self, sizehint=None):
1809         """Read multiple lines."""
1810         result = []
1811         for line in self:
1812             result.append(line)
1813             if sizehint is not None:
1814                 sizehint -= len(line)
1815                 if sizehint <= 0:
1816                     break
1817         return result
1818
1819     def __iter__(self):
1820         """Iterate over lines."""
1821         return iter(self.readline, "")
1822
1823     def tell(self):
1824         """Return the position."""
1825         return self._pos
1826
1827     def seek(self, offset, whence=0):
1828         """Change position."""
1829         if whence == 1:
1830             self._file.seek(self._pos)
1831         self._file.seek(offset, whence)
1832         self._pos = self._file.tell()
1833
1834     def close(self):
1835         """Close the file."""
1836         del self._file
1837
1838     def _read(self, size, read_method):
1839         """Read size bytes using read_method."""
1840         if size is None:
1841             size = -1
1842         self._file.seek(self._pos)
1843         result = read_method(size)
1844         self._pos = self._file.tell()
1845         return result
1846
1847
1848 class _PartialFile(_ProxyFile):
1849     """A read-only wrapper of part of a file."""
1850
1851     def __init__(self, f, start=None, stop=None):
1852         """Initialize a _PartialFile."""
1853         _ProxyFile.__init__(self, f, start)
1854         self._start = start
1855         self._stop = stop
1856
1857     def tell(self):
1858         """Return the position with respect to start."""
1859         return _ProxyFile.tell(self) - self._start
1860
1861     def seek(self, offset, whence=0):
1862         """Change position, possibly with respect to start or stop."""
1863         if whence == 0:
1864             self._pos = self._start
1865             whence = 1
1866         elif whence == 2:
1867             self._pos = self._stop
1868             whence = 1
1869         _ProxyFile.seek(self, offset, whence)
1870
1871     def _read(self, size, read_method):
1872         """Read size bytes using read_method, honoring start and stop."""
1873         remaining = self._stop - self._pos
1874         if remaining <= 0:
1875             return ''
1876         if size is None or size < 0 or size > remaining:
1877             size = remaining
1878         return _ProxyFile._read(self, size, read_method)
1879
1880
1881 def _lock_file(f, dotlock=True):
1882     """Lock file f using lockf and dot locking."""
1883     dotlock_done = False
1884     try:
1885         if fcntl:
1886             try:
1887                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1888             except IOError, e:
1889                 if e.errno in (errno.EAGAIN, errno.EACCES):
1890                     raise ExternalClashError('lockf: lock unavailable: %s' %
1891                                              f.name)
1892                 else:
1893                     raise
1894         if dotlock:
1895             try:
1896                 pre_lock = _create_temporary(f.name + '.lock')
1897                 pre_lock.close()
1898             except IOError, e:
1899                 if e.errno == errno.EACCES:
1900                     return  # Without write access, just skip dotlocking.
1901                 else:
1902                     raise
1903             try:
1904                 if hasattr(os, 'link'):
1905                     os.link(pre_lock.name, f.name + '.lock')
1906                     dotlock_done = True
1907                     os.unlink(pre_lock.name)
1908                 else:
1909                     os.rename(pre_lock.name, f.name + '.lock')
1910                     dotlock_done = True
1911             except OSError, e:
1912                 if e.errno == errno.EEXIST or \
1913                   (os.name == 'os2' and e.errno == errno.EACCES):
1914                     os.remove(pre_lock.name)
1915                     raise ExternalClashError('dot lock unavailable: %s' %
1916                                              f.name)
1917                 else:
1918                     raise
1919     except:
1920         if fcntl:
1921             fcntl.lockf(f, fcntl.LOCK_UN)
1922         if dotlock_done:
1923             os.remove(f.name + '.lock')
1924         raise
1925
1926 def _unlock_file(f):
1927     """Unlock file f using lockf and dot locking."""
1928     if fcntl:
1929         fcntl.lockf(f, fcntl.LOCK_UN)
1930     if os.path.exists(f.name + '.lock'):
1931         os.remove(f.name + '.lock')
1932
1933 def _create_carefully(path):
1934     """Create a file if it doesn't exist and open for reading and writing."""
1935     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
1936     try:
1937         return open(path, 'rb+')
1938     finally:
1939         os.close(fd)
1940
1941 def _create_temporary(path):
1942     """Create a temp file based on path and open for reading and writing."""
1943     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1944                                               socket.gethostname(),
1945                                               os.getpid()))
1946
1947 def _sync_flush(f):
1948     """Ensure changes to file f are physically on disk."""
1949     f.flush()
1950     if hasattr(os, 'fsync'):
1951         os.fsync(f.fileno())
1952
1953 def _sync_close(f):
1954     """Close file f, ensuring all changes are physically on disk."""
1955     _sync_flush(f)
1956     f.close()
1957
1958 ## Start: classes from the original module (for backward compatibility).
1959
1960 # Note that the Maildir class, whose name is unchanged, itself offers a next()
1961 # method for backward compatibility.
1962
1963 class _Mailbox:
1964
1965     def __init__(self, fp, factory=rfc822.Message):
1966         self.fp = fp
1967         self.seekp = 0
1968         self.factory = factory
1969
1970     def __iter__(self):
1971         return iter(self.next, None)
1972
1973     def next(self):
1974         while 1:
1975             self.fp.seek(self.seekp)
1976             try:
1977                 self._search_start()
1978             except EOFError:
1979                 self.seekp = self.fp.tell()
1980                 return None
1981             start = self.fp.tell()
1982             self._search_end()
1983             self.seekp = stop = self.fp.tell()
1984             if start != stop:
1985                 break
1986         return self.factory(_PartialFile(self.fp, start, stop))
1987
1988 # Recommended to use PortableUnixMailbox instead!
1989 class UnixMailbox(_Mailbox):
1990
1991     def _search_start(self):
1992         while 1:
1993             pos = self.fp.tell()
1994             line = self.fp.readline()
1995             if not line:
1996                 raise EOFError
1997             if line[:5] == 'From ' and self._isrealfromline(line):
1998                 self.fp.seek(pos)
1999                 return
2000
2001     def _search_end(self):
2002         self.fp.readline()      # Throw away header line
2003         while 1:
2004             pos = self.fp.tell()
2005             line = self.fp.readline()
2006             if not line:
2007                 return
2008             if line[:5] == 'From ' and self._isrealfromline(line):
2009                 self.fp.seek(pos)
2010                 return
2011
2012     # An overridable mechanism to test for From-line-ness.  You can either
2013     # specify a different regular expression or define a whole new
2014     # _isrealfromline() method.  Note that this only gets called for lines
2015     # starting with the 5 characters "From ".
2016     #
2017     # BAW: According to
2018     #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
2019     # the only portable, reliable way to find message delimiters in a BSD (i.e
2020     # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
2021     # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
2022     # like a good idea, in practice, there are too many variations for more
2023     # strict parsing of the line to be completely accurate.
2024     #
2025     # _strict_isrealfromline() is the old version which tries to do stricter
2026     # parsing of the From_ line.  _portable_isrealfromline() simply returns
2027     # true, since it's never called if the line doesn't already start with
2028     # "From ".
2029     #
2030     # This algorithm, and the way it interacts with _search_start() and
2031     # _search_end() may not be completely correct, because it doesn't check
2032     # that the two characters preceding "From " are \n\n or the beginning of
2033     # the file.  Fixing this would require a more extensive rewrite than is
2034     # necessary.  For convenience, we've added a PortableUnixMailbox class
2035     # which does no checking of the format of the 'From' line.
2036
2037     _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
2038                         r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
2039                         r"[^\s]*\s*"
2040                         "$")
2041     _regexp = None
2042
2043     def _strict_isrealfromline(self, line):
2044         if not self._regexp:
2045             import re
2046             self._regexp = re.compile(self._fromlinepattern)
2047         return self._regexp.match(line)
2048
2049     def _portable_isrealfromline(self, line):
2050         return True
2051
2052     _isrealfromline = _strict_isrealfromline
2053
2054
2055 class PortableUnixMailbox(UnixMailbox):
2056     _isrealfromline = UnixMailbox._portable_isrealfromline
2057
2058
2059 class MmdfMailbox(_Mailbox):
2060
2061     def _search_start(self):
2062         while 1:
2063             line = self.fp.readline()
2064             if not line:
2065                 raise EOFError
2066             if line[:5] == '\001\001\001\001\n':
2067                 return
2068
2069     def _search_end(self):
2070         while 1:
2071             pos = self.fp.tell()
2072             line = self.fp.readline()
2073             if not line:
2074                 return
2075             if line == '\001\001\001\001\n':
2076                 self.fp.seek(pos)
2077                 return
2078
2079
2080 class MHMailbox:
2081
2082     def __init__(self, dirname, factory=rfc822.Message):
2083         import re
2084         pat = re.compile('^[1-9][0-9]*$')
2085         self.dirname = dirname
2086         # the three following lines could be combined into:
2087         # list = map(long, filter(pat.match, os.listdir(self.dirname)))
2088         list = os.listdir(self.dirname)
2089         list = filter(pat.match, list)
2090         list = map(long, list)
2091         list.sort()
2092         # This only works in Python 1.6 or later;
2093         # before that str() added 'L':
2094         self.boxes = map(str, list)
2095         self.boxes.reverse()
2096         self.factory = factory
2097
2098     def __iter__(self):
2099         return iter(self.next, None)
2100
2101     def next(self):
2102         if not self.boxes:
2103             return None
2104         fn = self.boxes.pop()
2105         fp = open(os.path.join(self.dirname, fn))
2106         msg = self.factory(fp)
2107         try:
2108             msg._mh_msgno = fn
2109         except (AttributeError, TypeError):
2110             pass
2111         return msg
2112
2113
2114 class BabylMailbox(_Mailbox):
2115
2116     def _search_start(self):
2117         while 1:
2118             line = self.fp.readline()
2119             if not line:
2120                 raise EOFError
2121             if line == '*** EOOH ***\n':
2122                 return
2123
2124     def _search_end(self):
2125         while 1:
2126             pos = self.fp.tell()
2127             line = self.fp.readline()
2128             if not line:
2129                 return
2130             if line == '\037\014\n' or line == '\037':
2131                 self.fp.seek(pos)
2132                 return
2133
2134 ## End: classes from the original module (for backward compatibility).
2135
2136
2137 class Error(Exception):
2138     """Raised for module-specific errors."""
2139
2140 class NoSuchMailboxError(Error):
2141     """The specified mailbox does not exist and won't be created."""
2142
2143 class NotEmptyError(Error):
2144     """The specified mailbox is not empty and deletion was requested."""
2145
2146 class ExternalClashError(Error):
2147     """Another process caused an action to fail."""
2148
2149 class FormatError(Error):
2150     """A file appears to have an invalid format."""