Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.message
  20 import email.generator
  21 import io
  22 try:
  23     if sys.platform == 'os2emx':
  24         # OS/2 EMX fcntl() not adequate
  25         raise ImportError
  26     import fcntl
  27 except ImportError:
  28     fcntl = None
  29
  30 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  31             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  32             'BabylMessage', 'MMDFMessage']
  33
  34 class Mailbox:
  35     """A group of messages in a particular place."""
  36
  37     def __init__(self, path, factory=None, create=True):
  38         """Initialize a Mailbox instance."""
  39         self._path = os.path.abspath(os.path.expanduser(path))
  40         self._factory = factory
  41
  42     def add(self, message):
  43         """Add message and return assigned key."""
  44         raise NotImplementedError('Method must be implemented by subclass')
  45
  46     def remove(self, key):
  47         """Remove the keyed message; raise KeyError if it doesn't exist."""
  48         raise NotImplementedError('Method must be implemented by subclass')
  49
  50     def __delitem__(self, key):
  51         self.remove(key)
  52
  53     def discard(self, key):
  54         """If the keyed message exists, remove it."""
  55         try:
  56             self.remove(key)
  57         except KeyError:
  58             pass
  59
  60     def __setitem__(self, key, message):
  61         """Replace the keyed message; raise KeyError if it doesn't exist."""
  62         raise NotImplementedError('Method must be implemented by subclass')
  63
  64     def get(self, key, default=None):
  65         """Return the keyed message, or default if it doesn't exist."""
  66         try:
  67             return self.__getitem__(key)
  68         except KeyError:
  69             return default
  70
  71     def __getitem__(self, key):
  72         """Return the keyed message; raise KeyError if it doesn't exist."""
  73         if not self._factory:
  74             return self.get_message(key)
  75         else:
  76             return self._factory(self.get_file(key))
  77
  78     def get_message(self, key):
  79         """Return a Message representation or raise a KeyError."""
  80         raise NotImplementedError('Method must be implemented by subclass')
  81
  82     def get_string(self, key):
  83         """Return a string representation or raise a KeyError."""
  84         raise NotImplementedError('Method must be implemented by subclass')
  85
  86     def get_file(self, key):
  87         """Return a file-like representation or raise a KeyError."""
  88         raise NotImplementedError('Method must be implemented by subclass')
  89
  90     def iterkeys(self):
  91         """Return an iterator over keys."""
  92         raise NotImplementedError('Method must be implemented by subclass')
  93
  94     def keys(self):
  95         """Return a list of keys."""
  96         return list(self.iterkeys())
  97
  98     def itervalues(self):
  99         """Return an iterator over all messages."""
 100         for key in self.keys():
 101             try:
 102                 value = self[key]
 103             except KeyError:
 104                 continue
 105             yield value
 106
 107     def __iter__(self):
 108         return self.itervalues()
 109
 110     def values(self):
 111         """Return a list of messages. Memory intensive."""
 112         return list(self.itervalues())
 113
 114     def iteritems(self):
 115         """Return an iterator over (key, message) tuples."""
 116         for key in self.keys():
 117             try:
 118                 value = self[key]
 119             except KeyError:
 120                 continue
 121             yield (key, value)
 122
 123     def items(self):
 124         """Return a list of (key, message) tuples. Memory intensive."""
 125         return list(self.iteritems())
 126
 127     def __contains__(self, key):
 128         """Return True if the keyed message exists, False otherwise."""
 129         raise NotImplementedError('Method must be implemented by subclass')
 130
 131     def __len__(self):
 132         """Return a count of messages in the mailbox."""
 133         raise NotImplementedError('Method must be implemented by subclass')
 134
 135     def clear(self):
 136         """Delete all messages."""
 137         for key in self.keys():
 138             self.discard(key)
 139
 140     def pop(self, key, default=None):
 141         """Delete the keyed message and return it, or default."""
 142         try:
 143             result = self[key]
 144         except KeyError:
 145             return default
 146         self.discard(key)
 147         return result
 148
 149     def popitem(self):
 150         """Delete an arbitrary (key, message) pair and return it."""
 151         for key in self.keys():
 152             return (key, self.pop(key))     # This is only run once.
 153         else:
 154             raise KeyError('No messages in mailbox')
 155
 156     def update(self, arg=None):
 157         """Change the messages that correspond to certain keys."""
 158         if hasattr(arg, 'iteritems'):
 159             source = arg.items()
 160         elif hasattr(arg, 'items'):
 161             source = arg.items()
 162         else:
 163             source = arg
 164         bad_key = False
 165         for key, message in source:
 166             try:
 167                 self[key] = message
 168             except KeyError:
 169                 bad_key = True
 170         if bad_key:
 171             raise KeyError('No message with key(s)')
 172
 173     def flush(self):
 174         """Write any pending changes to the disk."""
 175         raise NotImplementedError('Method must be implemented by subclass')
 176
 177     def lock(self):
 178         """Lock the mailbox."""
 179         raise NotImplementedError('Method must be implemented by subclass')
 180
 181     def unlock(self):
 182         """Unlock the mailbox if it is locked."""
 183         raise NotImplementedError('Method must be implemented by subclass')
 184
 185     def close(self):
 186         """Flush and close the mailbox."""
 187         raise NotImplementedError('Method must be implemented by subclass')
 188
 189     def _dump_message(self, message, target, mangle_from_=False):
 190         # This assumes the target file is open in *text* mode with the
 191         # desired encoding and newline setting.
 192         """Dump message contents to target file."""
 193         if isinstance(message, email.message.Message):
 194             buffer = io.StringIO()
 195             gen = email.generator.Generator(buffer, mangle_from_, 0)
 196             gen.flatten(message)
 197             buffer.seek(0)
 198             data = buffer.read()
 199             ##data = data.replace('\n', os.linesep)
 200             target.write(data)
 201         elif isinstance(message, str):
 202             if mangle_from_:
 203                 message = message.replace('\nFrom ', '\n>From ')
 204             ##message = message.replace('\n', os.linesep)
 205             target.write(message)
 206         elif hasattr(message, 'read'):
 207             while True:
 208                 line = message.readline()
 209                 if not line:
 210                     break
 211                 if mangle_from_ and line.startswith('From '):
 212                     line = '>From ' + line[5:]
 213                 ##line = line.replace('\n', os.linesep)
 214                 target.write(line)
 215         else:
 216             raise TypeError('Invalid message type: %s' % type(message))
 217
 218
 219 class Maildir(Mailbox):
 220     """A qmail-style Maildir mailbox."""
 221
 222     colon = ':'
 223
 224     def __init__(self, dirname, factory=None, create=True):
 225         """Initialize a Maildir instance."""
 226         Mailbox.__init__(self, dirname, factory, create)
 227         if not os.path.exists(self._path):
 228             if create:
 229                 os.mkdir(self._path, 0o700)
 230                 os.mkdir(os.path.join(self._path, 'tmp'), 0o700)
 231                 os.mkdir(os.path.join(self._path, 'new'), 0o700)
 232                 os.mkdir(os.path.join(self._path, 'cur'), 0o700)
 233             else:
 234                 raise NoSuchMailboxError(self._path)
 235         self._toc = {}
 236         self._last_read = None          # Records last time we read cur/new
 237         # NOTE: we manually invalidate _last_read each time we do any
 238         # modifications ourselves, otherwise we might get tripped up by
 239         # bogus mtime behaviour on some systems (see issue #6896).
 240
 241     def add(self, message):
 242         """Add message and return assigned key."""
 243         tmp_file = self._create_tmp()
 244         try:
 245             self._dump_message(message, tmp_file)
 246         finally:
 247             _sync_close(tmp_file)
 248         if isinstance(message, MaildirMessage):
 249             subdir = message.get_subdir()
 250             suffix = self.colon + message.get_info()
 251             if suffix == self.colon:
 252                 suffix = ''
 253         else:
 254             subdir = 'new'
 255             suffix = ''
 256         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 257         dest = os.path.join(self._path, subdir, uniq + suffix)
 258         try:
 259             if hasattr(os, 'link'):
 260                 os.link(tmp_file.name, dest)
 261                 os.remove(tmp_file.name)
 262             else:
 263                 os.rename(tmp_file.name, dest)
 264         except OSError as e:
 265             os.remove(tmp_file.name)
 266             if e.errno == errno.EEXIST:
 267                 raise ExternalClashError('Name clash with existing message: %s'
 268                                          % dest)
 269             else:
 270                 raise
 271         if isinstance(message, MaildirMessage):
 272             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 273         # Invalidate cached toc
 274         self._last_read = None
 275         return uniq
 276
 277     def remove(self, key):
 278         """Remove the keyed message; raise KeyError if it doesn't exist."""
 279         os.remove(os.path.join(self._path, self._lookup(key)))
 280         # Invalidate cached toc (only on success)
 281         self._last_read = None
 282
 283     def discard(self, key):
 284         """If the keyed message exists, remove it."""
 285         # This overrides an inapplicable implementation in the superclass.
 286         try:
 287             self.remove(key)
 288         except KeyError:
 289             pass
 290         except OSError as e:
 291             if e.errno != errno.ENOENT:
 292                 raise
 293
 294     def __setitem__(self, key, message):
 295         """Replace the keyed message; raise KeyError if it doesn't exist."""
 296         old_subpath = self._lookup(key)
 297         temp_key = self.add(message)
 298         temp_subpath = self._lookup(temp_key)
 299         if isinstance(message, MaildirMessage):
 300             # temp's subdir and suffix were specified by message.
 301             dominant_subpath = temp_subpath
 302         else:
 303             # temp's subdir and suffix were defaults from add().
 304             dominant_subpath = old_subpath
 305         subdir = os.path.dirname(dominant_subpath)
 306         if self.colon in dominant_subpath:
 307             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 308         else:
 309             suffix = ''
 310         self.discard(key)
 311         new_path = os.path.join(self._path, subdir, key + suffix)
 312         os.rename(os.path.join(self._path, temp_subpath), new_path)
 313         if isinstance(message, MaildirMessage):
 314             os.utime(new_path, (os.path.getatime(new_path),
 315                                 message.get_date()))
 316         # Invalidate cached toc
 317         self._last_read = None
 318
 319     def get_message(self, key):
 320         """Return a Message representation or raise a KeyError."""
 321         subpath = self._lookup(key)
 322         f = open(os.path.join(self._path, subpath), 'r', newline='')
 323         try:
 324             if self._factory:
 325                 msg = self._factory(f)
 326             else:
 327                 msg = MaildirMessage(f)
 328         finally:
 329             f.close()
 330         subdir, name = os.path.split(subpath)
 331         msg.set_subdir(subdir)
 332         if self.colon in name:
 333             msg.set_info(name.split(self.colon)[-1])
 334         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 335         return msg
 336
 337     def get_string(self, key):
 338         """Return a string representation or raise a KeyError."""
 339         f = open(os.path.join(self._path, self._lookup(key)), 'r', newline='')
 340         try:
 341             return f.read()
 342         finally:
 343             f.close()
 344
 345     def get_file(self, key):
 346         """Return a file-like representation or raise a KeyError."""
 347         f = open(os.path.join(self._path, self._lookup(key)), 'r', newline='')
 348         return _ProxyFile(f)
 349
 350     def iterkeys(self):
 351         """Return an iterator over keys."""
 352         self._refresh()
 353         for key in self._toc:
 354             try:
 355                 self._lookup(key)
 356             except KeyError:
 357                 continue
 358             yield key
 359
 360     def __contains__(self, key):
 361         """Return True if the keyed message exists, False otherwise."""
 362         self._refresh()
 363         return key in self._toc
 364
 365     def __len__(self):
 366         """Return a count of messages in the mailbox."""
 367         self._refresh()
 368         return len(self._toc)
 369
 370     def flush(self):
 371         """Write any pending changes to disk."""
 372         # Maildir changes are always written immediately, so there's nothing
 373         # to do except invalidate our cached toc.
 374         self._last_read = None
 375
 376     def lock(self):
 377         """Lock the mailbox."""
 378         return
 379
 380     def unlock(self):
 381         """Unlock the mailbox if it is locked."""
 382         return
 383
 384     def close(self):
 385         """Flush and close the mailbox."""
 386         return
 387
 388     def list_folders(self):
 389         """Return a list of folder names."""
 390         result = []
 391         for entry in os.listdir(self._path):
 392             if len(entry) > 1 and entry[0] == '.' and \
 393                os.path.isdir(os.path.join(self._path, entry)):
 394                 result.append(entry[1:])
 395         return result
 396
 397     def get_folder(self, folder):
 398         """Return a Maildir instance for the named folder."""
 399         return Maildir(os.path.join(self._path, '.' + folder),
 400                        factory=self._factory,
 401                        create=False)
 402
 403     def add_folder(self, folder):
 404         """Create a folder and return a Maildir instance representing it."""
 405         path = os.path.join(self._path, '.' + folder)
 406         result = Maildir(path, factory=self._factory)
 407         maildirfolder_path = os.path.join(path, 'maildirfolder')
 408         if not os.path.exists(maildirfolder_path):
 409             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
 410                 0o666))
 411         return result
 412
 413     def remove_folder(self, folder):
 414         """Delete the named folder, which must be empty."""
 415         path = os.path.join(self._path, '.' + folder)
 416         for entry in os.listdir(os.path.join(path, 'new')) + \
 417                      os.listdir(os.path.join(path, 'cur')):
 418             if len(entry) < 1 or entry[0] != '.':
 419                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 420         for entry in os.listdir(path):
 421             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 422                os.path.isdir(os.path.join(path, entry)):
 423                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 424                                     (folder, entry))
 425         for root, dirs, files in os.walk(path, topdown=False):
 426             for entry in files:
 427                 os.remove(os.path.join(root, entry))
 428             for entry in dirs:
 429                 os.rmdir(os.path.join(root, entry))
 430         os.rmdir(path)
 431
 432     def clean(self):
 433         """Delete old files in "tmp"."""
 434         now = time.time()
 435         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 436             path = os.path.join(self._path, 'tmp', entry)
 437             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 438                 os.remove(path)
 439
 440     _count = 1  # This is used to generate unique file names.
 441
 442     def _create_tmp(self):
 443         """Create a file in the tmp subdirectory and open and return it."""
 444         now = time.time()
 445         hostname = socket.gethostname()
 446         if '/' in hostname:
 447             hostname = hostname.replace('/', r'\057')
 448         if ':' in hostname:
 449             hostname = hostname.replace(':', r'\072')
 450         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 451                                     Maildir._count, hostname)
 452         path = os.path.join(self._path, 'tmp', uniq)
 453         try:
 454             os.stat(path)
 455         except OSError as e:
 456             if e.errno == errno.ENOENT:
 457                 Maildir._count += 1
 458                 try:
 459                     return _create_carefully(path)
 460                 except OSError as e:
 461                     if e.errno != errno.EEXIST:
 462                         raise
 463             else:
 464                 raise
 465
 466         # Fall through to here if stat succeeded or open raised EEXIST.
 467         raise ExternalClashError('Name clash prevented file creation: %s' %
 468                                  path)
 469
 470     def _refresh(self):
 471         """Update table of contents mapping."""
 472         new_mtime = os.path.getmtime(os.path.join(self._path, 'new'))
 473         cur_mtime = os.path.getmtime(os.path.join(self._path, 'cur'))
 474
 475         if (self._last_read is not None and
 476             new_mtime <= self._last_read and cur_mtime <= self._last_read):
 477             return
 478
 479         self._toc = {}
 480         def update_dir (subdir):
 481             path = os.path.join(self._path, subdir)
 482             for entry in os.listdir(path):
 483                 p = os.path.join(path, entry)
 484                 if os.path.isdir(p):
 485                     continue
 486                 uniq = entry.split(self.colon)[0]
 487                 self._toc[uniq] = os.path.join(subdir, entry)
 488
 489         update_dir('new')
 490         update_dir('cur')
 491
 492         # We record the current time - 1sec so that, if _refresh() is called
 493         # again in the same second, we will always re-read the mailbox
 494         # just in case it's been modified.  (os.path.mtime() only has
 495         # 1sec resolution.)  This results in a few unnecessary re-reads
 496         # when _refresh() is called multiple times in the same second,
 497         # but once the clock ticks over, we will only re-read as needed.
 498         now = int(time.time() - 1)
 499         self._last_read = time.time() - 1
 500
 501     def _lookup(self, key):
 502         """Use TOC to return subpath for given key, or raise a KeyError."""
 503         try:
 504             if os.path.exists(os.path.join(self._path, self._toc[key])):
 505                 return self._toc[key]
 506         except KeyError:
 507             pass
 508         self._refresh()
 509         try:
 510             return self._toc[key]
 511         except KeyError:
 512             raise KeyError('No message with key: %s' % key)
 513
 514     # This method is for backward compatibility only.
 515     def next(self):
 516         """Return the next message in a one-time iteration."""
 517         if not hasattr(self, '_onetime_keys'):
 518             self._onetime_keys = iter(self.keys())
 519         while True:
 520             try:
 521                 return self[next(self._onetime_keys)]
 522             except StopIteration:
 523                 return None
 524             except KeyError:
 525                 continue
 526
 527
 528 class _singlefileMailbox(Mailbox):
 529     """A single-file mailbox."""
 530
 531     def __init__(self, path, factory=None, create=True):
 532         """Initialize a single-file mailbox."""
 533         Mailbox.__init__(self, path, factory, create)
 534         try:
 535             f = open(self._path, 'r+', newline='')
 536         except IOError as e:
 537             if e.errno == errno.ENOENT:
 538                 if create:
 539                     f = open(self._path, 'w+', newline='')
 540                 else:
 541                     raise NoSuchMailboxError(self._path)
 542             elif e.errno == errno.EACCES:
 543                 f = open(self._path, 'r', newline='')
 544             else:
 545                 raise
 546         self._file = f
 547         self._toc = None
 548         self._next_key = 0
 549         self._pending = False   # No changes require rewriting the file.
 550         self._locked = False
 551         self._file_length = None        # Used to record mailbox size
 552
 553     def add(self, message):
 554         """Add message and return assigned key."""
 555         self._lookup()
 556         self._toc[self._next_key] = self._append_message(message)
 557         self._next_key += 1
 558         self._pending = True
 559         return self._next_key - 1
 560
 561     def remove(self, key):
 562         """Remove the keyed message; raise KeyError if it doesn't exist."""
 563         self._lookup(key)
 564         del self._toc[key]
 565         self._pending = True
 566
 567     def __setitem__(self, key, message):
 568         """Replace the keyed message; raise KeyError if it doesn't exist."""
 569         self._lookup(key)
 570         self._toc[key] = self._append_message(message)
 571         self._pending = True
 572
 573     def iterkeys(self):
 574         """Return an iterator over keys."""
 575         self._lookup()
 576         for key in self._toc.keys():
 577             yield key
 578
 579     def __contains__(self, key):
 580         """Return True if the keyed message exists, False otherwise."""
 581         self._lookup()
 582         return key in self._toc
 583
 584     def __len__(self):
 585         """Return a count of messages in the mailbox."""
 586         self._lookup()
 587         return len(self._toc)
 588
 589     def lock(self):
 590         """Lock the mailbox."""
 591         if not self._locked:
 592             _lock_file(self._file)
 593             self._locked = True
 594
 595     def unlock(self):
 596         """Unlock the mailbox if it is locked."""
 597         if self._locked:
 598             _unlock_file(self._file)
 599             self._locked = False
 600
 601     def flush(self):
 602         """Write any pending changes to disk."""
 603         if not self._pending:
 604             return
 605
 606         # In order to be writing anything out at all, self._toc must
 607         # already have been generated (and presumably has been modified
 608         # by adding or deleting an item).
 609         assert self._toc is not None
 610
 611         # Check length of self._file; if it's changed, some other process
 612         # has modified the mailbox since we scanned it.
 613         self._file.seek(0, 2)
 614         cur_len = self._file.tell()
 615         if cur_len != self._file_length:
 616             raise ExternalClashError('Size of mailbox file changed '
 617                                      '(expected %i, found %i)' %
 618                                      (self._file_length, cur_len))
 619
 620         new_file = _create_temporary(self._path)
 621         try:
 622             new_toc = {}
 623             self._pre_mailbox_hook(new_file)
 624             for key in sorted(self._toc.keys()):
 625                 start, stop = self._toc[key]
 626                 self._file.seek(start)
 627                 self._pre_message_hook(new_file)
 628                 new_start = new_file.tell()
 629                 while True:
 630                     buffer = self._file.read(min(4096,
 631                                                  stop - self._file.tell()))
 632                     if not buffer:
 633                         break
 634                     new_file.write(buffer)
 635                 new_toc[key] = (new_start, new_file.tell())
 636                 self._post_message_hook(new_file)
 637         except:
 638             new_file.close()
 639             os.remove(new_file.name)
 640             raise
 641         _sync_close(new_file)
 642         # self._file is about to get replaced, so no need to sync.
 643         self._file.close()
 644         try:
 645             os.rename(new_file.name, self._path)
 646         except OSError as e:
 647             if e.errno == errno.EEXIST or \
 648               (os.name == 'os2' and e.errno == errno.EACCES):
 649                 os.remove(self._path)
 650                 os.rename(new_file.name, self._path)
 651             else:
 652                 raise
 653         self._file = open(self._path, 'rb+')
 654         self._toc = new_toc
 655         self._pending = False
 656         if self._locked:
 657             _lock_file(self._file, dotlock=False)
 658
 659     def _pre_mailbox_hook(self, f):
 660         """Called before writing the mailbox to file f."""
 661         return
 662
 663     def _pre_message_hook(self, f):
 664         """Called before writing each message to file f."""
 665         return
 666
 667     def _post_message_hook(self, f):
 668         """Called after writing each message to file f."""
 669         return
 670
 671     def close(self):
 672         """Flush and close the mailbox."""
 673         self.flush()
 674         if self._locked:
 675             self.unlock()
 676         self._file.close()  # Sync has been done by self.flush() above.
 677
 678     def _lookup(self, key=None):
 679         """Return (start, stop) or raise KeyError."""
 680         if self._toc is None:
 681             self._generate_toc()
 682         if key is not None:
 683             try:
 684                 return self._toc[key]
 685             except KeyError:
 686                 raise KeyError('No message with key: %s' % key)
 687
 688     def _append_message(self, message):
 689         """Append message to mailbox and return (start, stop) offsets."""
 690         self._file.seek(0, 2)
 691         self._pre_message_hook(self._file)
 692         offsets = self._install_message(message)
 693         self._post_message_hook(self._file)
 694         self._file.flush()
 695         self._file_length = self._file.tell()  # Record current length of mailbox
 696         return offsets
 697
 698
 699
 700 class _mboxMMDF(_singlefileMailbox):
 701     """An mbox or MMDF mailbox."""
 702
 703     _mangle_from_ = True
 704
 705     def get_message(self, key):
 706         """Return a Message representation or raise a KeyError."""
 707         start, stop = self._lookup(key)
 708         self._file.seek(start)
 709         from_line = self._file.readline().replace(os.linesep, '')
 710         string = self._file.read(stop - self._file.tell())
 711         msg = self._message_factory(string.replace(os.linesep, '\n'))
 712         msg.set_from(from_line[5:])
 713         return msg
 714
 715     def get_string(self, key, from_=False):
 716         """Return a string representation or raise a KeyError."""
 717         start, stop = self._lookup(key)
 718         self._file.seek(start)
 719         if not from_:
 720             self._file.readline()
 721         string = self._file.read(stop - self._file.tell())
 722         return string.replace(os.linesep, '\n')
 723
 724     def get_file(self, key, from_=False):
 725         """Return a file-like representation or raise a KeyError."""
 726         start, stop = self._lookup(key)
 727         self._file.seek(start)
 728         if not from_:
 729             self._file.readline()
 730         return _PartialFile(self._file, self._file.tell(), stop)
 731
 732     def _install_message(self, message):
 733         """Format a message and blindly write to self._file."""
 734         from_line = None
 735         if isinstance(message, str) and message.startswith('From '):
 736             newline = message.find('\n')
 737             if newline != -1:
 738                 from_line = message[:newline]
 739                 message = message[newline + 1:]
 740             else:
 741                 from_line = message
 742                 message = ''
 743         elif isinstance(message, _mboxMMDFMessage):
 744             from_line = 'From ' + message.get_from()
 745         elif isinstance(message, email.message.Message):
 746             from_line = message.get_unixfrom()  # May be None.
 747         if from_line is None:
 748             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 749         start = self._file.tell()
 750         self._file.write(from_line + os.linesep)
 751         self._dump_message(message, self._file, self._mangle_from_)
 752         stop = self._file.tell()
 753         return (start, stop)
 754
 755
 756 class mbox(_mboxMMDF):
 757     """A classic mbox mailbox."""
 758
 759     _mangle_from_ = True
 760
 761     def __init__(self, path, factory=None, create=True):
 762         """Initialize an mbox mailbox."""
 763         self._message_factory = mboxMessage
 764         _mboxMMDF.__init__(self, path, factory, create)
 765
 766     def _pre_message_hook(self, f):
 767         """Called before writing each message to file f."""
 768         if f.tell() != 0:
 769             f.write(os.linesep)
 770
 771     def _generate_toc(self):
 772         """Generate key-to-(start, stop) table of contents."""
 773         starts, stops = [], []
 774         self._file.seek(0)
 775         while True:
 776             line_pos = self._file.tell()
 777             line = self._file.readline()
 778             if line.startswith('From '):
 779                 if len(stops) < len(starts):
 780                     stops.append(line_pos - len(os.linesep))
 781                 starts.append(line_pos)
 782             elif not line:
 783                 stops.append(line_pos)
 784                 break
 785         self._toc = dict(enumerate(zip(starts, stops)))
 786         self._next_key = len(self._toc)
 787         self._file_length = self._file.tell()
 788
 789
 790 class MMDF(_mboxMMDF):
 791     """An MMDF mailbox."""
 792
 793     def __init__(self, path, factory=None, create=True):
 794         """Initialize an MMDF mailbox."""
 795         self._message_factory = MMDFMessage
 796         _mboxMMDF.__init__(self, path, factory, create)
 797
 798     def _pre_message_hook(self, f):
 799         """Called before writing each message to file f."""
 800         f.write('\001\001\001\001' + os.linesep)
 801
 802     def _post_message_hook(self, f):
 803         """Called after writing each message to file f."""
 804         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 805
 806     def _generate_toc(self):
 807         """Generate key-to-(start, stop) table of contents."""
 808         starts, stops = [], []
 809         self._file.seek(0)
 810         next_pos = 0
 811         while True:
 812             line_pos = next_pos
 813             line = self._file.readline()
 814             next_pos = self._file.tell()
 815             if line.startswith('\001\001\001\001' + os.linesep):
 816                 starts.append(next_pos)
 817                 while True:
 818                     line_pos = next_pos
 819                     line = self._file.readline()
 820                     next_pos = self._file.tell()
 821                     if line == '\001\001\001\001' + os.linesep:
 822                         stops.append(line_pos - len(os.linesep))
 823                         break
 824                     elif not line:
 825                         stops.append(line_pos)
 826                         break
 827             elif not line:
 828                 break
 829         self._toc = dict(enumerate(zip(starts, stops)))
 830         self._next_key = len(self._toc)
 831         self._file.seek(0, 2)
 832         self._file_length = self._file.tell()
 833
 834
 835 class MH(Mailbox):
 836     """An MH mailbox."""
 837
 838     def __init__(self, path, factory=None, create=True):
 839         """Initialize an MH instance."""
 840         Mailbox.__init__(self, path, factory, create)
 841         if not os.path.exists(self._path):
 842             if create:
 843                 os.mkdir(self._path, 0o700)
 844                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 845                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600))
 846             else:
 847                 raise NoSuchMailboxError(self._path)
 848         self._locked = False
 849
 850     def add(self, message):
 851         """Add message and return assigned key."""
 852         keys = self.keys()
 853         if len(keys) == 0:
 854             new_key = 1
 855         else:
 856             new_key = max(keys) + 1
 857         new_path = os.path.join(self._path, str(new_key))
 858         f = _create_carefully(new_path)
 859         try:
 860             if self._locked:
 861                 _lock_file(f)
 862             try:
 863                 self._dump_message(message, f)
 864                 if isinstance(message, MHMessage):
 865                     self._dump_sequences(message, new_key)
 866             finally:
 867                 if self._locked:
 868                     _unlock_file(f)
 869         finally:
 870             _sync_close(f)
 871         return new_key
 872
 873     def remove(self, key):
 874         """Remove the keyed message; raise KeyError if it doesn't exist."""
 875         path = os.path.join(self._path, str(key))
 876         try:
 877             f = open(path, 'rb+')
 878         except IOError as e:
 879             if e.errno == errno.ENOENT:
 880                 raise KeyError('No message with key: %s' % key)
 881             else:
 882                 raise
 883         try:
 884             if self._locked:
 885                 _lock_file(f)
 886             try:
 887                 f.close()
 888                 os.remove(os.path.join(self._path, str(key)))
 889             finally:
 890                 if self._locked:
 891                     _unlock_file(f)
 892         finally:
 893             f.close()
 894
 895     def __setitem__(self, key, message):
 896         """Replace the keyed message; raise KeyError if it doesn't exist."""
 897         path = os.path.join(self._path, str(key))
 898         try:
 899             f = open(path, 'r+', newline='')
 900         except IOError as e:
 901             if e.errno == errno.ENOENT:
 902                 raise KeyError('No message with key: %s' % key)
 903             else:
 904                 raise
 905         try:
 906             if self._locked:
 907                 _lock_file(f)
 908             try:
 909                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 910                 self._dump_message(message, f)
 911                 if isinstance(message, MHMessage):
 912                     self._dump_sequences(message, key)
 913             finally:
 914                 if self._locked:
 915                     _unlock_file(f)
 916         finally:
 917             _sync_close(f)
 918
 919     def get_message(self, key):
 920         """Return a Message representation or raise a KeyError."""
 921         try:
 922             if self._locked:
 923                 f = open(os.path.join(self._path, str(key)), 'r+', newline='')
 924             else:
 925                 f = open(os.path.join(self._path, str(key)), 'r', newline='')
 926         except IOError as e:
 927             if e.errno == errno.ENOENT:
 928                 raise KeyError('No message with key: %s' % key)
 929             else:
 930                 raise
 931         try:
 932             if self._locked:
 933                 _lock_file(f)
 934             try:
 935                 msg = MHMessage(f)
 936             finally:
 937                 if self._locked:
 938                     _unlock_file(f)
 939         finally:
 940             f.close()
 941         for name, key_list in self.get_sequences().items():
 942             if key in key_list:
 943                 msg.add_sequence(name)
 944         return msg
 945
 946     def get_string(self, key):
 947         """Return a string representation or raise a KeyError."""
 948         try:
 949             if self._locked:
 950                 f = open(os.path.join(self._path, str(key)), 'r+', newline='')
 951             else:
 952                 f = open(os.path.join(self._path, str(key)), 'r', newline='')
 953         except IOError as e:
 954             if e.errno == errno.ENOENT:
 955                 raise KeyError('No message with key: %s' % key)
 956             else:
 957                 raise
 958         try:
 959             if self._locked:
 960                 _lock_file(f)
 961             try:
 962                 return f.read()
 963             finally:
 964                 if self._locked:
 965                     _unlock_file(f)
 966         finally:
 967             f.close()
 968
 969     def get_file(self, key):
 970         """Return a file-like representation or raise a KeyError."""
 971         try:
 972             f = open(os.path.join(self._path, str(key)), 'r', newline='')
 973         except IOError as e:
 974             if e.errno == errno.ENOENT:
 975                 raise KeyError('No message with key: %s' % key)
 976             else:
 977                 raise
 978         return _ProxyFile(f)
 979
 980     def iterkeys(self):
 981         """Return an iterator over keys."""
 982         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 983                                       if entry.isdigit()))
 984
 985     def __contains__(self, key):
 986         """Return True if the keyed message exists, False otherwise."""
 987         return os.path.exists(os.path.join(self._path, str(key)))
 988
 989     def __len__(self):
 990         """Return a count of messages in the mailbox."""
 991         return len(list(self.keys()))
 992
 993     def lock(self):
 994         """Lock the mailbox."""
 995         if not self._locked:
 996             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
 997             _lock_file(self._file)
 998             self._locked = True
 999
1000     def unlock(self):
1001         """Unlock the mailbox if it is locked."""
1002         if self._locked:
1003             _unlock_file(self._file)
1004             _sync_close(self._file)
1005             del self._file
1006             self._locked = False
1007
1008     def flush(self):
1009         """Write any pending changes to the disk."""
1010         return
1011
1012     def close(self):
1013         """Flush and close the mailbox."""
1014         if self._locked:
1015             self.unlock()
1016
1017     def list_folders(self):
1018         """Return a list of folder names."""
1019         result = []
1020         for entry in os.listdir(self._path):
1021             if os.path.isdir(os.path.join(self._path, entry)):
1022                 result.append(entry)
1023         return result
1024
1025     def get_folder(self, folder):
1026         """Return an MH instance for the named folder."""
1027         return MH(os.path.join(self._path, folder),
1028                   factory=self._factory, create=False)
1029
1030     def add_folder(self, folder):
1031         """Create a folder and return an MH instance representing it."""
1032         return MH(os.path.join(self._path, folder),
1033                   factory=self._factory)
1034
1035     def remove_folder(self, folder):
1036         """Delete the named folder, which must be empty."""
1037         path = os.path.join(self._path, folder)
1038         entries = os.listdir(path)
1039         if entries == ['.mh_sequences']:
1040             os.remove(os.path.join(path, '.mh_sequences'))
1041         elif entries == []:
1042             pass
1043         else:
1044             raise NotEmptyError('Folder not empty: %s' % self._path)
1045         os.rmdir(path)
1046
1047     def get_sequences(self):
1048         """Return a name-to-key-list dictionary to define each sequence."""
1049         results = {}
1050         f = open(os.path.join(self._path, '.mh_sequences'), 'r', newline='')
1051         try:
1052             all_keys = set(self.keys())
1053             for line in f:
1054                 try:
1055                     name, contents = line.split(':')
1056                     keys = set()
1057                     for spec in contents.split():
1058                         if spec.isdigit():
1059                             keys.add(int(spec))
1060                         else:
1061                             start, stop = (int(x) for x in spec.split('-'))
1062                             keys.update(range(start, stop + 1))
1063                     results[name] = [key for key in sorted(keys) \
1064                                          if key in all_keys]
1065                     if len(results[name]) == 0:
1066                         del results[name]
1067                 except ValueError:
1068                     raise FormatError('Invalid sequence specification: %s' %
1069                                       line.rstrip())
1070         finally:
1071             f.close()
1072         return results
1073
1074     def set_sequences(self, sequences):
1075         """Set sequences using the given name-to-key-list dictionary."""
1076         f = open(os.path.join(self._path, '.mh_sequences'), 'r+', newline='')
1077         try:
1078             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1079             for name, keys in sequences.items():
1080                 if len(keys) == 0:
1081                     continue
1082                 f.write('%s:' % name)
1083                 prev = None
1084                 completing = False
1085                 for key in sorted(set(keys)):
1086                     if key - 1 == prev:
1087                         if not completing:
1088                             completing = True
1089                             f.write('-')
1090                     elif completing:
1091                         completing = False
1092                         f.write('%s %s' % (prev, key))
1093                     else:
1094                         f.write(' %s' % key)
1095                     prev = key
1096                 if completing:
1097                     f.write(str(prev) + '\n')
1098                 else:
1099                     f.write('\n')
1100         finally:
1101             _sync_close(f)
1102
1103     def pack(self):
1104         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1105         sequences = self.get_sequences()
1106         prev = 0
1107         changes = []
1108         for key in self.keys():
1109             if key - 1 != prev:
1110                 changes.append((key, prev + 1))
1111                 if hasattr(os, 'link'):
1112                     os.link(os.path.join(self._path, str(key)),
1113                             os.path.join(self._path, str(prev + 1)))
1114                     os.unlink(os.path.join(self._path, str(key)))
1115                 else:
1116                     os.rename(os.path.join(self._path, str(key)),
1117                               os.path.join(self._path, str(prev + 1)))
1118             prev += 1
1119         self._next_key = prev + 1
1120         if len(changes) == 0:
1121             return
1122         for name, key_list in sequences.items():
1123             for old, new in changes:
1124                 if old in key_list:
1125                     key_list[key_list.index(old)] = new
1126         self.set_sequences(sequences)
1127
1128     def _dump_sequences(self, message, key):
1129         """Inspect a new MHMessage and update sequences appropriately."""
1130         pending_sequences = message.get_sequences()
1131         all_sequences = self.get_sequences()
1132         for name, key_list in all_sequences.items():
1133             if name in pending_sequences:
1134                 key_list.append(key)
1135             elif key in key_list:
1136                 del key_list[key_list.index(key)]
1137         for sequence in pending_sequences:
1138             if sequence not in all_sequences:
1139                 all_sequences[sequence] = [key]
1140         self.set_sequences(all_sequences)
1141
1142
1143 class Babyl(_singlefileMailbox):
1144     """An Rmail-style Babyl mailbox."""
1145
1146     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1147                                  'forwarded', 'edited', 'resent'))
1148
1149     def __init__(self, path, factory=None, create=True):
1150         """Initialize a Babyl mailbox."""
1151         _singlefileMailbox.__init__(self, path, factory, create)
1152         self._labels = {}
1153
1154     def add(self, message):
1155         """Add message and return assigned key."""
1156         key = _singlefileMailbox.add(self, message)
1157         if isinstance(message, BabylMessage):
1158             self._labels[key] = message.get_labels()
1159         return key
1160
1161     def remove(self, key):
1162         """Remove the keyed message; raise KeyError if it doesn't exist."""
1163         _singlefileMailbox.remove(self, key)
1164         if key in self._labels:
1165             del self._labels[key]
1166
1167     def __setitem__(self, key, message):
1168         """Replace the keyed message; raise KeyError if it doesn't exist."""
1169         _singlefileMailbox.__setitem__(self, key, message)
1170         if isinstance(message, BabylMessage):
1171             self._labels[key] = message.get_labels()
1172
1173     def get_message(self, key):
1174         """Return a Message representation or raise a KeyError."""
1175         start, stop = self._lookup(key)
1176         self._file.seek(start)
1177         self._file.readline()   # Skip '1,' line specifying labels.
1178         original_headers = io.StringIO()
1179         while True:
1180             line = self._file.readline()
1181             if line == '*** EOOH ***' + os.linesep or not line:
1182                 break
1183             original_headers.write(line.replace(os.linesep, '\n'))
1184         visible_headers = io.StringIO()
1185         while True:
1186             line = self._file.readline()
1187             if line == os.linesep or not line:
1188                 break
1189             visible_headers.write(line.replace(os.linesep, '\n'))
1190         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1191                                                                  '\n')
1192         msg = BabylMessage(original_headers.getvalue() + body)
1193         msg.set_visible(visible_headers.getvalue())
1194         if key in self._labels:
1195             msg.set_labels(self._labels[key])
1196         return msg
1197
1198     def get_string(self, key):
1199         """Return a string representation or raise a KeyError."""
1200         start, stop = self._lookup(key)
1201         self._file.seek(start)
1202         self._file.readline()   # Skip '1,' line specifying labels.
1203         original_headers = io.StringIO()
1204         while True:
1205             line = self._file.readline()
1206             if line == '*** EOOH ***' + os.linesep or not line:
1207                 break
1208             original_headers.write(line.replace(os.linesep, '\n'))
1209         while True:
1210             line = self._file.readline()
1211             if line == os.linesep or not line:
1212                 break
1213         return original_headers.getvalue() + \
1214                self._file.read(stop - self._file.tell()).replace(os.linesep,
1215                                                                  '\n')
1216
1217     def get_file(self, key):
1218         """Return a file-like representation or raise a KeyError."""
1219         return io.StringIO(self.get_string(key).replace('\n',
1220                                                               os.linesep))
1221
1222     def get_labels(self):
1223         """Return a list of user-defined labels in the mailbox."""
1224         self._lookup()
1225         labels = set()
1226         for label_list in self._labels.values():
1227             labels.update(label_list)
1228         labels.difference_update(self._special_labels)
1229         return list(labels)
1230
1231     def _generate_toc(self):
1232         """Generate key-to-(start, stop) table of contents."""
1233         starts, stops = [], []
1234         self._file.seek(0)
1235         next_pos = 0
1236         label_lists = []
1237         while True:
1238             line_pos = next_pos
1239             line = self._file.readline()
1240             next_pos = self._file.tell()
1241             if line == '\037\014' + os.linesep:
1242                 if len(stops) < len(starts):
1243                     stops.append(line_pos - len(os.linesep))
1244                 starts.append(next_pos)
1245                 labels = [label.strip() for label
1246                                         in self._file.readline()[1:].split(',')
1247                                         if label.strip()]
1248                 label_lists.append(labels)
1249             elif line == '\037' or line == '\037' + os.linesep:
1250                 if len(stops) < len(starts):
1251                     stops.append(line_pos - len(os.linesep))
1252             elif not line:
1253                 stops.append(line_pos - len(os.linesep))
1254                 break
1255         self._toc = dict(enumerate(zip(starts, stops)))
1256         self._labels = dict(enumerate(label_lists))
1257         self._next_key = len(self._toc)
1258         self._file.seek(0, 2)
1259         self._file_length = self._file.tell()
1260
1261     def _pre_mailbox_hook(self, f):
1262         """Called before writing the mailbox to file f."""
1263         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1264                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1265                  os.linesep))
1266
1267     def _pre_message_hook(self, f):
1268         """Called before writing each message to file f."""
1269         f.write('\014' + os.linesep)
1270
1271     def _post_message_hook(self, f):
1272         """Called after writing each message to file f."""
1273         f.write(os.linesep + '\037')
1274
1275     def _install_message(self, message):
1276         """Write message contents and return (start, stop)."""
1277         start = self._file.tell()
1278         if isinstance(message, BabylMessage):
1279             special_labels = []
1280             labels = []
1281             for label in message.get_labels():
1282                 if label in self._special_labels:
1283                     special_labels.append(label)
1284                 else:
1285                     labels.append(label)
1286             self._file.write('1')
1287             for label in special_labels:
1288                 self._file.write(', ' + label)
1289             self._file.write(',,')
1290             for label in labels:
1291                 self._file.write(' ' + label + ',')
1292             self._file.write(os.linesep)
1293         else:
1294             self._file.write('1,,' + os.linesep)
1295         if isinstance(message, email.message.Message):
1296             orig_buffer = io.StringIO()
1297             orig_generator = email.generator.Generator(orig_buffer, False, 0)
1298             orig_generator.flatten(message)
1299             orig_buffer.seek(0)
1300             while True:
1301                 line = orig_buffer.readline()
1302                 self._file.write(line.replace('\n', os.linesep))
1303                 if line == '\n' or not line:
1304                     break
1305             self._file.write('*** EOOH ***' + os.linesep)
1306             if isinstance(message, BabylMessage):
1307                 vis_buffer = io.StringIO()
1308                 vis_generator = email.generator.Generator(vis_buffer, False, 0)
1309                 vis_generator.flatten(message.get_visible())
1310                 while True:
1311                     line = vis_buffer.readline()
1312                     self._file.write(line.replace('\n', os.linesep))
1313                     if line == '\n' or not line:
1314                         break
1315             else:
1316                 orig_buffer.seek(0)
1317                 while True:
1318                     line = orig_buffer.readline()
1319                     self._file.write(line.replace('\n', os.linesep))
1320                     if line == '\n' or not line:
1321                         break
1322             while True:
1323                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1324                 if not buffer:
1325                     break
1326                 self._file.write(buffer.replace('\n', os.linesep))
1327         elif isinstance(message, str):
1328             body_start = message.find('\n\n') + 2
1329             if body_start - 2 != -1:
1330                 self._file.write(message[:body_start].replace('\n',
1331                                                               os.linesep))
1332                 self._file.write('*** EOOH ***' + os.linesep)
1333                 self._file.write(message[:body_start].replace('\n',
1334                                                               os.linesep))
1335                 self._file.write(message[body_start:].replace('\n',
1336                                                               os.linesep))
1337             else:
1338                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1339                 self._file.write(message.replace('\n', os.linesep))
1340         elif hasattr(message, 'readline'):
1341             original_pos = message.tell()
1342             first_pass = True
1343             while True:
1344                 line = message.readline()
1345                 self._file.write(line.replace('\n', os.linesep))
1346                 if line == '\n' or not line:
1347                     self._file.write('*** EOOH ***' + os.linesep)
1348                     if first_pass:
1349                         first_pass = False
1350                         message.seek(original_pos)
1351                     else:
1352                         break
1353             while True:
1354                 buffer = message.read(4096)     # Buffer size is arbitrary.
1355                 if not buffer:
1356                     break
1357                 self._file.write(buffer.replace('\n', os.linesep))
1358         else:
1359             raise TypeError('Invalid message type: %s' % type(message))
1360         stop = self._file.tell()
1361         return (start, stop)
1362
1363
1364 class Message(email.message.Message):
1365     """Message with mailbox-format-specific properties."""
1366
1367     def __init__(self, message=None):
1368         """Initialize a Message instance."""
1369         if isinstance(message, email.message.Message):
1370             self._become_message(copy.deepcopy(message))
1371             if isinstance(message, Message):
1372                 message._explain_to(self)
1373         elif isinstance(message, str):
1374             self._become_message(email.message_from_string(message))
1375         elif hasattr(message, "read"):
1376             self._become_message(email.message_from_file(message))
1377         elif message is None:
1378             email.message.Message.__init__(self)
1379         else:
1380             raise TypeError('Invalid message type: %s' % type(message))
1381
1382     def _become_message(self, message):
1383         """Assume the non-format-specific state of message."""
1384         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1385                      'preamble', 'epilogue', 'defects', '_default_type'):
1386             self.__dict__[name] = message.__dict__[name]
1387
1388     def _explain_to(self, message):
1389         """Copy format-specific state to message insofar as possible."""
1390         if isinstance(message, Message):
1391             return  # There's nothing format-specific to explain.
1392         else:
1393             raise TypeError('Cannot convert to specified type')
1394
1395
1396 class MaildirMessage(Message):
1397     """Message with Maildir-specific properties."""
1398
1399     def __init__(self, message=None):
1400         """Initialize a MaildirMessage instance."""
1401         self._subdir = 'new'
1402         self._info = ''
1403         self._date = time.time()
1404         Message.__init__(self, message)
1405
1406     def get_subdir(self):
1407         """Return 'new' or 'cur'."""
1408         return self._subdir
1409
1410     def set_subdir(self, subdir):
1411         """Set subdir to 'new' or 'cur'."""
1412         if subdir == 'new' or subdir == 'cur':
1413             self._subdir = subdir
1414         else:
1415             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1416
1417     def get_flags(self):
1418         """Return as a string the flags that are set."""
1419         if self._info.startswith('2,'):
1420             return self._info[2:]
1421         else:
1422             return ''
1423
1424     def set_flags(self, flags):
1425         """Set the given flags and unset all others."""
1426         self._info = '2,' + ''.join(sorted(flags))
1427
1428     def add_flag(self, flag):
1429         """Set the given flag(s) without changing others."""
1430         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1431
1432     def remove_flag(self, flag):
1433         """Unset the given string flag(s) without changing others."""
1434         if self.get_flags():
1435             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1436
1437     def get_date(self):
1438         """Return delivery date of message, in seconds since the epoch."""
1439         return self._date
1440
1441     def set_date(self, date):
1442         """Set delivery date of message, in seconds since the epoch."""
1443         try:
1444             self._date = float(date)
1445         except ValueError:
1446             raise TypeError("can't convert to float: %s" % date)
1447
1448     def get_info(self):
1449         """Get the message's "info" as a string."""
1450         return self._info
1451
1452     def set_info(self, info):
1453         """Set the message's "info" string."""
1454         if isinstance(info, str):
1455             self._info = info
1456         else:
1457             raise TypeError('info must be a string: %s' % type(info))
1458
1459     def _explain_to(self, message):
1460         """Copy Maildir-specific state to message insofar as possible."""
1461         if isinstance(message, MaildirMessage):
1462             message.set_flags(self.get_flags())
1463             message.set_subdir(self.get_subdir())
1464             message.set_date(self.get_date())
1465         elif isinstance(message, _mboxMMDFMessage):
1466             flags = set(self.get_flags())
1467             if 'S' in flags:
1468                 message.add_flag('R')
1469             if self.get_subdir() == 'cur':
1470                 message.add_flag('O')
1471             if 'T' in flags:
1472                 message.add_flag('D')
1473             if 'F' in flags:
1474                 message.add_flag('F')
1475             if 'R' in flags:
1476                 message.add_flag('A')
1477             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1478         elif isinstance(message, MHMessage):
1479             flags = set(self.get_flags())
1480             if 'S' not in flags:
1481                 message.add_sequence('unseen')
1482             if 'R' in flags:
1483                 message.add_sequence('replied')
1484             if 'F' in flags:
1485                 message.add_sequence('flagged')
1486         elif isinstance(message, BabylMessage):
1487             flags = set(self.get_flags())
1488             if 'S' not in flags:
1489                 message.add_label('unseen')
1490             if 'T' in flags:
1491                 message.add_label('deleted')
1492             if 'R' in flags:
1493                 message.add_label('answered')
1494             if 'P' in flags:
1495                 message.add_label('forwarded')
1496         elif isinstance(message, Message):
1497             pass
1498         else:
1499             raise TypeError('Cannot convert to specified type: %s' %
1500                             type(message))
1501
1502
1503 class _mboxMMDFMessage(Message):
1504     """Message with mbox- or MMDF-specific properties."""
1505
1506     def __init__(self, message=None):
1507         """Initialize an mboxMMDFMessage instance."""
1508         self.set_from('MAILER-DAEMON', True)
1509         if isinstance(message, email.message.Message):
1510             unixfrom = message.get_unixfrom()
1511             if unixfrom is not None and unixfrom.startswith('From '):
1512                 self.set_from(unixfrom[5:])
1513         Message.__init__(self, message)
1514
1515     def get_from(self):
1516         """Return contents of "From " line."""
1517         return self._from
1518
1519     def set_from(self, from_, time_=None):
1520         """Set "From " line, formatting and appending time_ if specified."""
1521         if time_ is not None:
1522             if time_ is True:
1523                 time_ = time.gmtime()
1524             from_ += ' ' + time.asctime(time_)
1525         self._from = from_
1526
1527     def get_flags(self):
1528         """Return as a string the flags that are set."""
1529         return self.get('Status', '') + self.get('X-Status', '')
1530
1531     def set_flags(self, flags):
1532         """Set the given flags and unset all others."""
1533         flags = set(flags)
1534         status_flags, xstatus_flags = '', ''
1535         for flag in ('R', 'O'):
1536             if flag in flags:
1537                 status_flags += flag
1538                 flags.remove(flag)
1539         for flag in ('D', 'F', 'A'):
1540             if flag in flags:
1541                 xstatus_flags += flag
1542                 flags.remove(flag)
1543         xstatus_flags += ''.join(sorted(flags))
1544         try:
1545             self.replace_header('Status', status_flags)
1546         except KeyError:
1547             self.add_header('Status', status_flags)
1548         try:
1549             self.replace_header('X-Status', xstatus_flags)
1550         except KeyError:
1551             self.add_header('X-Status', xstatus_flags)
1552
1553     def add_flag(self, flag):
1554         """Set the given flag(s) without changing others."""
1555         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1556
1557     def remove_flag(self, flag):
1558         """Unset the given string flag(s) without changing others."""
1559         if 'Status' in self or 'X-Status' in self:
1560             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1561
1562     def _explain_to(self, message):
1563         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1564         if isinstance(message, MaildirMessage):
1565             flags = set(self.get_flags())
1566             if 'O' in flags:
1567                 message.set_subdir('cur')
1568             if 'F' in flags:
1569                 message.add_flag('F')
1570             if 'A' in flags:
1571                 message.add_flag('R')
1572             if 'R' in flags:
1573                 message.add_flag('S')
1574             if 'D' in flags:
1575                 message.add_flag('T')
1576             del message['status']
1577             del message['x-status']
1578             maybe_date = ' '.join(self.get_from().split()[-5:])
1579             try:
1580                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1581                                                       '%a %b %d %H:%M:%S %Y')))
1582             except (ValueError, OverflowError):
1583                 pass
1584         elif isinstance(message, _mboxMMDFMessage):
1585             message.set_flags(self.get_flags())
1586             message.set_from(self.get_from())
1587         elif isinstance(message, MHMessage):
1588             flags = set(self.get_flags())
1589             if 'R' not in flags:
1590                 message.add_sequence('unseen')
1591             if 'A' in flags:
1592                 message.add_sequence('replied')
1593             if 'F' in flags:
1594                 message.add_sequence('flagged')
1595             del message['status']
1596             del message['x-status']
1597         elif isinstance(message, BabylMessage):
1598             flags = set(self.get_flags())
1599             if 'R' not in flags:
1600                 message.add_label('unseen')
1601             if 'D' in flags:
1602                 message.add_label('deleted')
1603             if 'A' in flags:
1604                 message.add_label('answered')
1605             del message['status']
1606             del message['x-status']
1607         elif isinstance(message, Message):
1608             pass
1609         else:
1610             raise TypeError('Cannot convert to specified type: %s' %
1611                             type(message))
1612
1613
1614 class mboxMessage(_mboxMMDFMessage):
1615     """Message with mbox-specific properties."""
1616
1617
1618 class MHMessage(Message):
1619     """Message with MH-specific properties."""
1620
1621     def __init__(self, message=None):
1622         """Initialize an MHMessage instance."""
1623         self._sequences = []
1624         Message.__init__(self, message)
1625
1626     def get_sequences(self):
1627         """Return a list of sequences that include the message."""
1628         return self._sequences[:]
1629
1630     def set_sequences(self, sequences):
1631         """Set the list of sequences that include the message."""
1632         self._sequences = list(sequences)
1633
1634     def add_sequence(self, sequence):
1635         """Add sequence to list of sequences including the message."""
1636         if isinstance(sequence, str):
1637             if not sequence in self._sequences:
1638                 self._sequences.append(sequence)
1639         else:
1640             raise TypeError('sequence must be a string: %s' % type(sequence))
1641
1642     def remove_sequence(self, sequence):
1643         """Remove sequence from the list of sequences including the message."""
1644         try:
1645             self._sequences.remove(sequence)
1646         except ValueError:
1647             pass
1648
1649     def _explain_to(self, message):
1650         """Copy MH-specific state to message insofar as possible."""
1651         if isinstance(message, MaildirMessage):
1652             sequences = set(self.get_sequences())
1653             if 'unseen' in sequences:
1654                 message.set_subdir('cur')
1655             else:
1656                 message.set_subdir('cur')
1657                 message.add_flag('S')
1658             if 'flagged' in sequences:
1659                 message.add_flag('F')
1660             if 'replied' in sequences:
1661                 message.add_flag('R')
1662         elif isinstance(message, _mboxMMDFMessage):
1663             sequences = set(self.get_sequences())
1664             if 'unseen' not in sequences:
1665                 message.add_flag('RO')
1666             else:
1667                 message.add_flag('O')
1668             if 'flagged' in sequences:
1669                 message.add_flag('F')
1670             if 'replied' in sequences:
1671                 message.add_flag('A')
1672         elif isinstance(message, MHMessage):
1673             for sequence in self.get_sequences():
1674                 message.add_sequence(sequence)
1675         elif isinstance(message, BabylMessage):
1676             sequences = set(self.get_sequences())
1677             if 'unseen' in sequences:
1678                 message.add_label('unseen')
1679             if 'replied' in sequences:
1680                 message.add_label('answered')
1681         elif isinstance(message, Message):
1682             pass
1683         else:
1684             raise TypeError('Cannot convert to specified type: %s' %
1685                             type(message))
1686
1687
1688 class BabylMessage(Message):
1689     """Message with Babyl-specific properties."""
1690
1691     def __init__(self, message=None):
1692         """Initialize an BabylMessage instance."""
1693         self._labels = []
1694         self._visible = Message()
1695         Message.__init__(self, message)
1696
1697     def get_labels(self):
1698         """Return a list of labels on the message."""
1699         return self._labels[:]
1700
1701     def set_labels(self, labels):
1702         """Set the list of labels on the message."""
1703         self._labels = list(labels)
1704
1705     def add_label(self, label):
1706         """Add label to list of labels on the message."""
1707         if isinstance(label, str):
1708             if label not in self._labels:
1709                 self._labels.append(label)
1710         else:
1711             raise TypeError('label must be a string: %s' % type(label))
1712
1713     def remove_label(self, label):
1714         """Remove label from the list of labels on the message."""
1715         try:
1716             self._labels.remove(label)
1717         except ValueError:
1718             pass
1719
1720     def get_visible(self):
1721         """Return a Message representation of visible headers."""
1722         return Message(self._visible)
1723
1724     def set_visible(self, visible):
1725         """Set the Message representation of visible headers."""
1726         self._visible = Message(visible)
1727
1728     def update_visible(self):
1729         """Update and/or sensibly generate a set of visible headers."""
1730         for header in self._visible.keys():
1731             if header in self:
1732                 self._visible.replace_header(header, self[header])
1733             else:
1734                 del self._visible[header]
1735         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1736             if header in self and header not in self._visible:
1737                 self._visible[header] = self[header]
1738
1739     def _explain_to(self, message):
1740         """Copy Babyl-specific state to message insofar as possible."""
1741         if isinstance(message, MaildirMessage):
1742             labels = set(self.get_labels())
1743             if 'unseen' in labels:
1744                 message.set_subdir('cur')
1745             else:
1746                 message.set_subdir('cur')
1747                 message.add_flag('S')
1748             if 'forwarded' in labels or 'resent' in labels:
1749                 message.add_flag('P')
1750             if 'answered' in labels:
1751                 message.add_flag('R')
1752             if 'deleted' in labels:
1753                 message.add_flag('T')
1754         elif isinstance(message, _mboxMMDFMessage):
1755             labels = set(self.get_labels())
1756             if 'unseen' not in labels:
1757                 message.add_flag('RO')
1758             else:
1759                 message.add_flag('O')
1760             if 'deleted' in labels:
1761                 message.add_flag('D')
1762             if 'answered' in labels:
1763                 message.add_flag('A')
1764         elif isinstance(message, MHMessage):
1765             labels = set(self.get_labels())
1766             if 'unseen' in labels:
1767                 message.add_sequence('unseen')
1768             if 'answered' in labels:
1769                 message.add_sequence('replied')
1770         elif isinstance(message, BabylMessage):
1771             message.set_visible(self.get_visible())
1772             for label in self.get_labels():
1773                 message.add_label(label)
1774         elif isinstance(message, Message):
1775             pass
1776         else:
1777             raise TypeError('Cannot convert to specified type: %s' %
1778                             type(message))
1779
1780
1781 class MMDFMessage(_mboxMMDFMessage):
1782     """Message with MMDF-specific properties."""
1783
1784
1785 class _ProxyFile:
1786     """A read-only wrapper of a file."""
1787
1788     def __init__(self, f, pos=None):
1789         """Initialize a _ProxyFile."""
1790         self._file = f
1791         if pos is None:
1792             self._pos = f.tell()
1793         else:
1794             self._pos = pos
1795
1796     def read(self, size=None):
1797         """Read bytes."""
1798         return self._read(size, self._file.read)
1799
1800     def readline(self, size=None):
1801         """Read a line."""
1802         return self._read(size, self._file.readline)
1803
1804     def readlines(self, sizehint=None):
1805         """Read multiple lines."""
1806         result = []
1807         for line in self:
1808             result.append(line)
1809             if sizehint is not None:
1810                 sizehint -= len(line)
1811                 if sizehint <= 0:
1812                     break
1813         return result
1814
1815     def __iter__(self):
1816         """Iterate over lines."""
1817         while True:
1818             line = self.readline()
1819             if not line:
1820                 raise StopIteration
1821             yield line
1822
1823     def tell(self):
1824         """Return the position."""
1825         return self._pos
1826
1827     def seek(self, offset, whence=0):
1828         """Change position."""
1829         if whence == 1:
1830             self._file.seek(self._pos)
1831         self._file.seek(offset, whence)
1832         self._pos = self._file.tell()
1833
1834     def close(self):
1835         """Close the file."""
1836         del self._file
1837
1838     def _read(self, size, read_method):
1839         """Read size bytes using read_method."""
1840         if size is None:
1841             size = -1
1842         self._file.seek(self._pos)
1843         result = read_method(size)
1844         self._pos = self._file.tell()
1845         return result
1846
1847
1848 class _PartialFile(_ProxyFile):
1849     """A read-only wrapper of part of a file."""
1850
1851     def __init__(self, f, start=None, stop=None):
1852         """Initialize a _PartialFile."""
1853         _ProxyFile.__init__(self, f, start)
1854         self._start = start
1855         self._stop = stop
1856
1857     def tell(self):
1858         """Return the position with respect to start."""
1859         return _ProxyFile.tell(self) - self._start
1860
1861     def seek(self, offset, whence=0):
1862         """Change position, possibly with respect to start or stop."""
1863         if whence == 0:
1864             self._pos = self._start
1865             whence = 1
1866         elif whence == 2:
1867             self._pos = self._stop
1868             whence = 1
1869         _ProxyFile.seek(self, offset, whence)
1870
1871     def _read(self, size, read_method):
1872         """Read size bytes using read_method, honoring start and stop."""
1873         remaining = self._stop - self._pos
1874         if remaining <= 0:
1875             return ''
1876         if size is None or size < 0 or size > remaining:
1877             size = remaining
1878         return _ProxyFile._read(self, size, read_method)
1879
1880
1881 def _lock_file(f, dotlock=True):
1882     """Lock file f using lockf and dot locking."""
1883     dotlock_done = False
1884     try:
1885         if fcntl:
1886             try:
1887                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1888             except IOError as e:
1889                 if e.errno in (errno.EAGAIN, errno.EACCES):
1890                     raise ExternalClashError('lockf: lock unavailable: %s' %
1891                                              f.name)
1892                 else:
1893                     raise
1894         if dotlock:
1895             try:
1896                 pre_lock = _create_temporary(f.name + '.lock')
1897                 pre_lock.close()
1898             except IOError as e:
1899                 if e.errno == errno.EACCES:
1900                     return  # Without write access, just skip dotlocking.
1901                 else:
1902                     raise
1903             try:
1904                 if hasattr(os, 'link'):
1905                     os.link(pre_lock.name, f.name + '.lock')
1906                     dotlock_done = True
1907                     os.unlink(pre_lock.name)
1908                 else:
1909                     os.rename(pre_lock.name, f.name + '.lock')
1910                     dotlock_done = True
1911             except OSError as e:
1912                 if e.errno == errno.EEXIST or \
1913                   (os.name == 'os2' and e.errno == errno.EACCES):
1914                     os.remove(pre_lock.name)
1915                     raise ExternalClashError('dot lock unavailable: %s' %
1916                                              f.name)
1917                 else:
1918                     raise
1919     except:
1920         if fcntl:
1921             fcntl.lockf(f, fcntl.LOCK_UN)
1922         if dotlock_done:
1923             os.remove(f.name + '.lock')
1924         raise
1925
1926 def _unlock_file(f):
1927     """Unlock file f using lockf and dot locking."""
1928     if fcntl:
1929         fcntl.lockf(f, fcntl.LOCK_UN)
1930     if os.path.exists(f.name + '.lock'):
1931         os.remove(f.name + '.lock')
1932
1933 def _create_carefully(path):
1934     """Create a file if it doesn't exist and open for reading and writing."""
1935     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666)
1936     try:
1937         return open(path, 'r+', newline='')
1938     finally:
1939         os.close(fd)
1940
1941 def _create_temporary(path):
1942     """Create a temp file based on path and open for reading and writing."""
1943     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1944                                               socket.gethostname(),
1945                                               os.getpid()))
1946
1947 def _sync_flush(f):
1948     """Ensure changes to file f are physically on disk."""
1949     f.flush()
1950     if hasattr(os, 'fsync'):
1951         os.fsync(f.fileno())
1952
1953 def _sync_close(f):
1954     """Close file f, ensuring all changes are physically on disk."""
1955     _sync_flush(f)
1956     f.close()
1957
1958
1959 class Error(Exception):
1960     """Raised for module-specific errors."""
1961
1962 class NoSuchMailboxError(Error):
1963     """The specified mailbox does not exist and won't be created."""
1964
1965 class NotEmptyError(Error):
1966     """The specified mailbox is not empty and deletion was requested."""
1967
1968 class ExternalClashError(Error):
1969     """Another process caused an action to fail."""
1970
1971 class FormatError(Error):
1972     """A file appears to have an invalid format."""