Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.message
  20 import email.generator
  21 import io
  22 try:
  23     if sys.platform == 'os2emx':
  24         # OS/2 EMX fcntl() not adequate
  25         raise ImportError
  26     import fcntl
  27 except ImportError:
  28     fcntl = None
  29
  30 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  31             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  32             'BabylMessage', 'MMDFMessage']
  33
  34 class Mailbox:
  35     """A group of messages in a particular place."""
  36
  37     def __init__(self, path, factory=None, create=True):
  38         """Initialize a Mailbox instance."""
  39         self._path = os.path.abspath(os.path.expanduser(path))
  40         self._factory = factory
  41
  42     def add(self, message):
  43         """Add message and return assigned key."""
  44         raise NotImplementedError('Method must be implemented by subclass')
  45
  46     def remove(self, key):
  47         """Remove the keyed message; raise KeyError if it doesn't exist."""
  48         raise NotImplementedError('Method must be implemented by subclass')
  49
  50     def __delitem__(self, key):
  51         self.remove(key)
  52
  53     def discard(self, key):
  54         """If the keyed message exists, remove it."""
  55         try:
  56             self.remove(key)
  57         except KeyError:
  58             pass
  59
  60     def __setitem__(self, key, message):
  61         """Replace the keyed message; raise KeyError if it doesn't exist."""
  62         raise NotImplementedError('Method must be implemented by subclass')
  63
  64     def get(self, key, default=None):
  65         """Return the keyed message, or default if it doesn't exist."""
  66         try:
  67             return self.__getitem__(key)
  68         except KeyError:
  69             return default
  70
  71     def __getitem__(self, key):
  72         """Return the keyed message; raise KeyError if it doesn't exist."""
  73         if not self._factory:
  74             return self.get_message(key)
  75         else:
  76             return self._factory(self.get_file(key))
  77
  78     def get_message(self, key):
  79         """Return a Message representation or raise a KeyError."""
  80         raise NotImplementedError('Method must be implemented by subclass')
  81
  82     def get_string(self, key):
  83         """Return a string representation or raise a KeyError."""
  84         raise NotImplementedError('Method must be implemented by subclass')
  85
  86     def get_file(self, key):
  87         """Return a file-like representation or raise a KeyError."""
  88         raise NotImplementedError('Method must be implemented by subclass')
  89
  90     def iterkeys(self):
  91         """Return an iterator over keys."""
  92         raise NotImplementedError('Method must be implemented by subclass')
  93
  94     def keys(self):
  95         """Return a list of keys."""
  96         return list(self.iterkeys())
  97
  98     def itervalues(self):
  99         """Return an iterator over all messages."""
 100         for key in self.keys():
 101             try:
 102                 value = self[key]
 103             except KeyError:
 104                 continue
 105             yield value
 106
 107     def __iter__(self):
 108         return self.itervalues()
 109
 110     def values(self):
 111         """Return a list of messages. Memory intensive."""
 112         return list(self.itervalues())
 113
 114     def iteritems(self):
 115         """Return an iterator over (key, message) tuples."""
 116         for key in self.keys():
 117             try:
 118                 value = self[key]
 119             except KeyError:
 120                 continue
 121             yield (key, value)
 122
 123     def items(self):
 124         """Return a list of (key, message) tuples. Memory intensive."""
 125         return list(self.iteritems())
 126
 127     def __contains__(self, key):
 128         """Return True if the keyed message exists, False otherwise."""
 129         raise NotImplementedError('Method must be implemented by subclass')
 130
 131     def __len__(self):
 132         """Return a count of messages in the mailbox."""
 133         raise NotImplementedError('Method must be implemented by subclass')
 134
 135     def clear(self):
 136         """Delete all messages."""
 137         for key in self.keys():
 138             self.discard(key)
 139
 140     def pop(self, key, default=None):
 141         """Delete the keyed message and return it, or default."""
 142         try:
 143             result = self[key]
 144         except KeyError:
 145             return default
 146         self.discard(key)
 147         return result
 148
 149     def popitem(self):
 150         """Delete an arbitrary (key, message) pair and return it."""
 151         for key in self.keys():
 152             return (key, self.pop(key))     # This is only run once.
 153         else:
 154             raise KeyError('No messages in mailbox')
 155
 156     def update(self, arg=None):
 157         """Change the messages that correspond to certain keys."""
 158         if hasattr(arg, 'iteritems'):
 159             source = arg.items()
 160         elif hasattr(arg, 'items'):
 161             source = arg.items()
 162         else:
 163             source = arg
 164         bad_key = False
 165         for key, message in source:
 166             try:
 167                 self[key] = message
 168             except KeyError:
 169                 bad_key = True
 170         if bad_key:
 171             raise KeyError('No message with key(s)')
 172
 173     def flush(self):
 174         """Write any pending changes to the disk."""
 175         raise NotImplementedError('Method must be implemented by subclass')
 176
 177     def lock(self):
 178         """Lock the mailbox."""
 179         raise NotImplementedError('Method must be implemented by subclass')
 180
 181     def unlock(self):
 182         """Unlock the mailbox if it is locked."""
 183         raise NotImplementedError('Method must be implemented by subclass')
 184
 185     def close(self):
 186         """Flush and close the mailbox."""
 187         raise NotImplementedError('Method must be implemented by subclass')
 188
 189     def _dump_message(self, message, target, mangle_from_=False):
 190         # This assumes the target file is open in *text* mode with the
 191         # desired encoding and newline setting.
 192         """Dump message contents to target file."""
 193         if isinstance(message, email.message.Message):
 194             buffer = io.StringIO()
 195             gen = email.generator.Generator(buffer, mangle_from_, 0)
 196             gen.flatten(message)
 197             buffer.seek(0)
 198             data = buffer.read()
 199             ##data = data.replace('\n', os.linesep)
 200             target.write(data)
 201         elif isinstance(message, str):
 202             if mangle_from_:
 203                 message = message.replace('\nFrom ', '\n>From ')
 204             ##message = message.replace('\n', os.linesep)
 205             target.write(message)
 206         elif hasattr(message, 'read'):
 207             while True:
 208                 line = message.readline()
 209                 if not line:
 210                     break
 211                 if mangle_from_ and line.startswith('From '):
 212                     line = '>From ' + line[5:]
 213                 ##line = line.replace('\n', os.linesep)
 214                 target.write(line)
 215         else:
 216             raise TypeError('Invalid message type: %s' % type(message))
 217
 218
 219 class Maildir(Mailbox):
 220     """A qmail-style Maildir mailbox."""
 221
 222     colon = ':'
 223
 224     def __init__(self, dirname, factory=None, create=True):
 225         """Initialize a Maildir instance."""
 226         Mailbox.__init__(self, dirname, factory, create)
 227         if not os.path.exists(self._path):
 228             if create:
 229                 os.mkdir(self._path, 0o700)
 230                 os.mkdir(os.path.join(self._path, 'tmp'), 0o700)
 231                 os.mkdir(os.path.join(self._path, 'new'), 0o700)
 232                 os.mkdir(os.path.join(self._path, 'cur'), 0o700)
 233             else:
 234                 raise NoSuchMailboxError(self._path)
 235         self._toc = {}
 236         self._last_read = None          # Records last time we read cur/new
 237
 238     def add(self, message):
 239         """Add message and return assigned key."""
 240         tmp_file = self._create_tmp()
 241         try:
 242             self._dump_message(message, tmp_file)
 243         finally:
 244             _sync_close(tmp_file)
 245         if isinstance(message, MaildirMessage):
 246             subdir = message.get_subdir()
 247             suffix = self.colon + message.get_info()
 248             if suffix == self.colon:
 249                 suffix = ''
 250         else:
 251             subdir = 'new'
 252             suffix = ''
 253         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 254         dest = os.path.join(self._path, subdir, uniq + suffix)
 255         try:
 256             if hasattr(os, 'link'):
 257                 os.link(tmp_file.name, dest)
 258                 os.remove(tmp_file.name)
 259             else:
 260                 os.rename(tmp_file.name, dest)
 261         except OSError as e:
 262             os.remove(tmp_file.name)
 263             if e.errno == errno.EEXIST:
 264                 raise ExternalClashError('Name clash with existing message: %s'
 265                                          % dest)
 266             else:
 267                 raise
 268         if isinstance(message, MaildirMessage):
 269             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 270         return uniq
 271
 272     def remove(self, key):
 273         """Remove the keyed message; raise KeyError if it doesn't exist."""
 274         os.remove(os.path.join(self._path, self._lookup(key)))
 275
 276     def discard(self, key):
 277         """If the keyed message exists, remove it."""
 278         # This overrides an inapplicable implementation in the superclass.
 279         try:
 280             self.remove(key)
 281         except KeyError:
 282             pass
 283         except OSError as e:
 284             if e.errno != errno.ENOENT:
 285                 raise
 286
 287     def __setitem__(self, key, message):
 288         """Replace the keyed message; raise KeyError if it doesn't exist."""
 289         old_subpath = self._lookup(key)
 290         temp_key = self.add(message)
 291         temp_subpath = self._lookup(temp_key)
 292         if isinstance(message, MaildirMessage):
 293             # temp's subdir and suffix were specified by message.
 294             dominant_subpath = temp_subpath
 295         else:
 296             # temp's subdir and suffix were defaults from add().
 297             dominant_subpath = old_subpath
 298         subdir = os.path.dirname(dominant_subpath)
 299         if self.colon in dominant_subpath:
 300             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 301         else:
 302             suffix = ''
 303         self.discard(key)
 304         new_path = os.path.join(self._path, subdir, key + suffix)
 305         os.rename(os.path.join(self._path, temp_subpath), new_path)
 306         if isinstance(message, MaildirMessage):
 307             os.utime(new_path, (os.path.getatime(new_path),
 308                                 message.get_date()))
 309
 310     def get_message(self, key):
 311         """Return a Message representation or raise a KeyError."""
 312         subpath = self._lookup(key)
 313         f = open(os.path.join(self._path, subpath), 'r', newline='')
 314         try:
 315             if self._factory:
 316                 msg = self._factory(f)
 317             else:
 318                 msg = MaildirMessage(f)
 319         finally:
 320             f.close()
 321         subdir, name = os.path.split(subpath)
 322         msg.set_subdir(subdir)
 323         if self.colon in name:
 324             msg.set_info(name.split(self.colon)[-1])
 325         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 326         return msg
 327
 328     def get_string(self, key):
 329         """Return a string representation or raise a KeyError."""
 330         f = open(os.path.join(self._path, self._lookup(key)), 'r', newline='')
 331         try:
 332             return f.read()
 333         finally:
 334             f.close()
 335
 336     def get_file(self, key):
 337         """Return a file-like representation or raise a KeyError."""
 338         f = open(os.path.join(self._path, self._lookup(key)), 'r', newline='')
 339         return _ProxyFile(f)
 340
 341     def iterkeys(self):
 342         """Return an iterator over keys."""
 343         self._refresh()
 344         for key in self._toc:
 345             try:
 346                 self._lookup(key)
 347             except KeyError:
 348                 continue
 349             yield key
 350
 351     def __contains__(self, key):
 352         """Return True if the keyed message exists, False otherwise."""
 353         self._refresh()
 354         return key in self._toc
 355
 356     def __len__(self):
 357         """Return a count of messages in the mailbox."""
 358         self._refresh()
 359         return len(self._toc)
 360
 361     def flush(self):
 362         """Write any pending changes to disk."""
 363         return  # Maildir changes are always written immediately.
 364
 365     def lock(self):
 366         """Lock the mailbox."""
 367         return
 368
 369     def unlock(self):
 370         """Unlock the mailbox if it is locked."""
 371         return
 372
 373     def close(self):
 374         """Flush and close the mailbox."""
 375         return
 376
 377     def list_folders(self):
 378         """Return a list of folder names."""
 379         result = []
 380         for entry in os.listdir(self._path):
 381             if len(entry) > 1 and entry[0] == '.' and \
 382                os.path.isdir(os.path.join(self._path, entry)):
 383                 result.append(entry[1:])
 384         return result
 385
 386     def get_folder(self, folder):
 387         """Return a Maildir instance for the named folder."""
 388         return Maildir(os.path.join(self._path, '.' + folder),
 389                        factory=self._factory,
 390                        create=False)
 391
 392     def add_folder(self, folder):
 393         """Create a folder and return a Maildir instance representing it."""
 394         path = os.path.join(self._path, '.' + folder)
 395         result = Maildir(path, factory=self._factory)
 396         maildirfolder_path = os.path.join(path, 'maildirfolder')
 397         if not os.path.exists(maildirfolder_path):
 398             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
 399                 0o666))
 400         return result
 401
 402     def remove_folder(self, folder):
 403         """Delete the named folder, which must be empty."""
 404         path = os.path.join(self._path, '.' + folder)
 405         for entry in os.listdir(os.path.join(path, 'new')) + \
 406                      os.listdir(os.path.join(path, 'cur')):
 407             if len(entry) < 1 or entry[0] != '.':
 408                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 409         for entry in os.listdir(path):
 410             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 411                os.path.isdir(os.path.join(path, entry)):
 412                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 413                                     (folder, entry))
 414         for root, dirs, files in os.walk(path, topdown=False):
 415             for entry in files:
 416                 os.remove(os.path.join(root, entry))
 417             for entry in dirs:
 418                 os.rmdir(os.path.join(root, entry))
 419         os.rmdir(path)
 420
 421     def clean(self):
 422         """Delete old files in "tmp"."""
 423         now = time.time()
 424         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 425             path = os.path.join(self._path, 'tmp', entry)
 426             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 427                 os.remove(path)
 428
 429     _count = 1  # This is used to generate unique file names.
 430
 431     def _create_tmp(self):
 432         """Create a file in the tmp subdirectory and open and return it."""
 433         now = time.time()
 434         hostname = socket.gethostname()
 435         if '/' in hostname:
 436             hostname = hostname.replace('/', r'\057')
 437         if ':' in hostname:
 438             hostname = hostname.replace(':', r'\072')
 439         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 440                                     Maildir._count, hostname)
 441         path = os.path.join(self._path, 'tmp', uniq)
 442         try:
 443             os.stat(path)
 444         except OSError as e:
 445             if e.errno == errno.ENOENT:
 446                 Maildir._count += 1
 447                 try:
 448                     return _create_carefully(path)
 449                 except OSError as e:
 450                     if e.errno != errno.EEXIST:
 451                         raise
 452             else:
 453                 raise
 454
 455         # Fall through to here if stat succeeded or open raised EEXIST.
 456         raise ExternalClashError('Name clash prevented file creation: %s' %
 457                                  path)
 458
 459     def _refresh(self):
 460         """Update table of contents mapping."""
 461         new_mtime = os.path.getmtime(os.path.join(self._path, 'new'))
 462         cur_mtime = os.path.getmtime(os.path.join(self._path, 'cur'))
 463
 464         if (self._last_read is not None and
 465             new_mtime <= self._last_read and cur_mtime <= self._last_read):
 466             return
 467
 468         self._toc = {}
 469         def update_dir (subdir):
 470             path = os.path.join(self._path, subdir)
 471             for entry in os.listdir(path):
 472                 p = os.path.join(path, entry)
 473                 if os.path.isdir(p):
 474                     continue
 475                 uniq = entry.split(self.colon)[0]
 476                 self._toc[uniq] = os.path.join(subdir, entry)
 477
 478         update_dir('new')
 479         update_dir('cur')
 480
 481         # We record the current time - 1sec so that, if _refresh() is called
 482         # again in the same second, we will always re-read the mailbox
 483         # just in case it's been modified.  (os.path.mtime() only has
 484         # 1sec resolution.)  This results in a few unnecessary re-reads
 485         # when _refresh() is called multiple times in the same second,
 486         # but once the clock ticks over, we will only re-read as needed.
 487         now = int(time.time() - 1)
 488         self._last_read = time.time() - 1
 489
 490     def _lookup(self, key):
 491         """Use TOC to return subpath for given key, or raise a KeyError."""
 492         try:
 493             if os.path.exists(os.path.join(self._path, self._toc[key])):
 494                 return self._toc[key]
 495         except KeyError:
 496             pass
 497         self._refresh()
 498         try:
 499             return self._toc[key]
 500         except KeyError:
 501             raise KeyError('No message with key: %s' % key)
 502
 503     # This method is for backward compatibility only.
 504     def next(self):
 505         """Return the next message in a one-time iteration."""
 506         if not hasattr(self, '_onetime_keys'):
 507             self._onetime_keys = iter(self.keys())
 508         while True:
 509             try:
 510                 return self[next(self._onetime_keys)]
 511             except StopIteration:
 512                 return None
 513             except KeyError:
 514                 continue
 515
 516
 517 class _singlefileMailbox(Mailbox):
 518     """A single-file mailbox."""
 519
 520     def __init__(self, path, factory=None, create=True):
 521         """Initialize a single-file mailbox."""
 522         Mailbox.__init__(self, path, factory, create)
 523         try:
 524             f = open(self._path, 'r+', newline='')
 525         except IOError as e:
 526             if e.errno == errno.ENOENT:
 527                 if create:
 528                     f = open(self._path, 'w+', newline='')
 529                 else:
 530                     raise NoSuchMailboxError(self._path)
 531             elif e.errno == errno.EACCES:
 532                 f = open(self._path, 'r', newline='')
 533             else:
 534                 raise
 535         self._file = f
 536         self._toc = None
 537         self._next_key = 0
 538         self._pending = False   # No changes require rewriting the file.
 539         self._locked = False
 540         self._file_length = None        # Used to record mailbox size
 541
 542     def add(self, message):
 543         """Add message and return assigned key."""
 544         self._lookup()
 545         self._toc[self._next_key] = self._append_message(message)
 546         self._next_key += 1
 547         self._pending = True
 548         return self._next_key - 1
 549
 550     def remove(self, key):
 551         """Remove the keyed message; raise KeyError if it doesn't exist."""
 552         self._lookup(key)
 553         del self._toc[key]
 554         self._pending = True
 555
 556     def __setitem__(self, key, message):
 557         """Replace the keyed message; raise KeyError if it doesn't exist."""
 558         self._lookup(key)
 559         self._toc[key] = self._append_message(message)
 560         self._pending = True
 561
 562     def iterkeys(self):
 563         """Return an iterator over keys."""
 564         self._lookup()
 565         for key in self._toc.keys():
 566             yield key
 567
 568     def __contains__(self, key):
 569         """Return True if the keyed message exists, False otherwise."""
 570         self._lookup()
 571         return key in self._toc
 572
 573     def __len__(self):
 574         """Return a count of messages in the mailbox."""
 575         self._lookup()
 576         return len(self._toc)
 577
 578     def lock(self):
 579         """Lock the mailbox."""
 580         if not self._locked:
 581             _lock_file(self._file)
 582             self._locked = True
 583
 584     def unlock(self):
 585         """Unlock the mailbox if it is locked."""
 586         if self._locked:
 587             _unlock_file(self._file)
 588             self._locked = False
 589
 590     def flush(self):
 591         """Write any pending changes to disk."""
 592         if not self._pending:
 593             return
 594
 595         # In order to be writing anything out at all, self._toc must
 596         # already have been generated (and presumably has been modified
 597         # by adding or deleting an item).
 598         assert self._toc is not None
 599
 600         # Check length of self._file; if it's changed, some other process
 601         # has modified the mailbox since we scanned it.
 602         self._file.seek(0, 2)
 603         cur_len = self._file.tell()
 604         if cur_len != self._file_length:
 605             raise ExternalClashError('Size of mailbox file changed '
 606                                      '(expected %i, found %i)' %
 607                                      (self._file_length, cur_len))
 608
 609         new_file = _create_temporary(self._path)
 610         try:
 611             new_toc = {}
 612             self._pre_mailbox_hook(new_file)
 613             for key in sorted(self._toc.keys()):
 614                 start, stop = self._toc[key]
 615                 self._file.seek(start)
 616                 self._pre_message_hook(new_file)
 617                 new_start = new_file.tell()
 618                 while True:
 619                     buffer = self._file.read(min(4096,
 620                                                  stop - self._file.tell()))
 621                     if not buffer:
 622                         break
 623                     new_file.write(buffer)
 624                 new_toc[key] = (new_start, new_file.tell())
 625                 self._post_message_hook(new_file)
 626         except:
 627             new_file.close()
 628             os.remove(new_file.name)
 629             raise
 630         _sync_close(new_file)
 631         # self._file is about to get replaced, so no need to sync.
 632         self._file.close()
 633         try:
 634             os.rename(new_file.name, self._path)
 635         except OSError as e:
 636             if e.errno == errno.EEXIST or \
 637               (os.name == 'os2' and e.errno == errno.EACCES):
 638                 os.remove(self._path)
 639                 os.rename(new_file.name, self._path)
 640             else:
 641                 raise
 642         self._file = open(self._path, 'rb+')
 643         self._toc = new_toc
 644         self._pending = False
 645         if self._locked:
 646             _lock_file(self._file, dotlock=False)
 647
 648     def _pre_mailbox_hook(self, f):
 649         """Called before writing the mailbox to file f."""
 650         return
 651
 652     def _pre_message_hook(self, f):
 653         """Called before writing each message to file f."""
 654         return
 655
 656     def _post_message_hook(self, f):
 657         """Called after writing each message to file f."""
 658         return
 659
 660     def close(self):
 661         """Flush and close the mailbox."""
 662         self.flush()
 663         if self._locked:
 664             self.unlock()
 665         self._file.close()  # Sync has been done by self.flush() above.
 666
 667     def _lookup(self, key=None):
 668         """Return (start, stop) or raise KeyError."""
 669         if self._toc is None:
 670             self._generate_toc()
 671         if key is not None:
 672             try:
 673                 return self._toc[key]
 674             except KeyError:
 675                 raise KeyError('No message with key: %s' % key)
 676
 677     def _append_message(self, message):
 678         """Append message to mailbox and return (start, stop) offsets."""
 679         self._file.seek(0, 2)
 680         self._pre_message_hook(self._file)
 681         offsets = self._install_message(message)
 682         self._post_message_hook(self._file)
 683         self._file.flush()
 684         self._file_length = self._file.tell()  # Record current length of mailbox
 685         return offsets
 686
 687
 688
 689 class _mboxMMDF(_singlefileMailbox):
 690     """An mbox or MMDF mailbox."""
 691
 692     _mangle_from_ = True
 693
 694     def get_message(self, key):
 695         """Return a Message representation or raise a KeyError."""
 696         start, stop = self._lookup(key)
 697         self._file.seek(start)
 698         from_line = self._file.readline().replace(os.linesep, '')
 699         string = self._file.read(stop - self._file.tell())
 700         msg = self._message_factory(string.replace(os.linesep, '\n'))
 701         msg.set_from(from_line[5:])
 702         return msg
 703
 704     def get_string(self, key, from_=False):
 705         """Return a string representation or raise a KeyError."""
 706         start, stop = self._lookup(key)
 707         self._file.seek(start)
 708         if not from_:
 709             self._file.readline()
 710         string = self._file.read(stop - self._file.tell())
 711         return string.replace(os.linesep, '\n')
 712
 713     def get_file(self, key, from_=False):
 714         """Return a file-like representation or raise a KeyError."""
 715         start, stop = self._lookup(key)
 716         self._file.seek(start)
 717         if not from_:
 718             self._file.readline()
 719         return _PartialFile(self._file, self._file.tell(), stop)
 720
 721     def _install_message(self, message):
 722         """Format a message and blindly write to self._file."""
 723         from_line = None
 724         if isinstance(message, str) and message.startswith('From '):
 725             newline = message.find('\n')
 726             if newline != -1:
 727                 from_line = message[:newline]
 728                 message = message[newline + 1:]
 729             else:
 730                 from_line = message
 731                 message = ''
 732         elif isinstance(message, _mboxMMDFMessage):
 733             from_line = 'From ' + message.get_from()
 734         elif isinstance(message, email.message.Message):
 735             from_line = message.get_unixfrom()  # May be None.
 736         if from_line is None:
 737             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 738         start = self._file.tell()
 739         self._file.write(from_line + os.linesep)
 740         self._dump_message(message, self._file, self._mangle_from_)
 741         stop = self._file.tell()
 742         return (start, stop)
 743
 744
 745 class mbox(_mboxMMDF):
 746     """A classic mbox mailbox."""
 747
 748     _mangle_from_ = True
 749
 750     def __init__(self, path, factory=None, create=True):
 751         """Initialize an mbox mailbox."""
 752         self._message_factory = mboxMessage
 753         _mboxMMDF.__init__(self, path, factory, create)
 754
 755     def _pre_message_hook(self, f):
 756         """Called before writing each message to file f."""
 757         if f.tell() != 0:
 758             f.write(os.linesep)
 759
 760     def _generate_toc(self):
 761         """Generate key-to-(start, stop) table of contents."""
 762         starts, stops = [], []
 763         self._file.seek(0)
 764         while True:
 765             line_pos = self._file.tell()
 766             line = self._file.readline()
 767             if line.startswith('From '):
 768                 if len(stops) < len(starts):
 769                     stops.append(line_pos - len(os.linesep))
 770                 starts.append(line_pos)
 771             elif not line:
 772                 stops.append(line_pos)
 773                 break
 774         self._toc = dict(enumerate(zip(starts, stops)))
 775         self._next_key = len(self._toc)
 776         self._file_length = self._file.tell()
 777
 778
 779 class MMDF(_mboxMMDF):
 780     """An MMDF mailbox."""
 781
 782     def __init__(self, path, factory=None, create=True):
 783         """Initialize an MMDF mailbox."""
 784         self._message_factory = MMDFMessage
 785         _mboxMMDF.__init__(self, path, factory, create)
 786
 787     def _pre_message_hook(self, f):
 788         """Called before writing each message to file f."""
 789         f.write('\001\001\001\001' + os.linesep)
 790
 791     def _post_message_hook(self, f):
 792         """Called after writing each message to file f."""
 793         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 794
 795     def _generate_toc(self):
 796         """Generate key-to-(start, stop) table of contents."""
 797         starts, stops = [], []
 798         self._file.seek(0)
 799         next_pos = 0
 800         while True:
 801             line_pos = next_pos
 802             line = self._file.readline()
 803             next_pos = self._file.tell()
 804             if line.startswith('\001\001\001\001' + os.linesep):
 805                 starts.append(next_pos)
 806                 while True:
 807                     line_pos = next_pos
 808                     line = self._file.readline()
 809                     next_pos = self._file.tell()
 810                     if line == '\001\001\001\001' + os.linesep:
 811                         stops.append(line_pos - len(os.linesep))
 812                         break
 813                     elif not line:
 814                         stops.append(line_pos)
 815                         break
 816             elif not line:
 817                 break
 818         self._toc = dict(enumerate(zip(starts, stops)))
 819         self._next_key = len(self._toc)
 820         self._file.seek(0, 2)
 821         self._file_length = self._file.tell()
 822
 823
 824 class MH(Mailbox):
 825     """An MH mailbox."""
 826
 827     def __init__(self, path, factory=None, create=True):
 828         """Initialize an MH instance."""
 829         Mailbox.__init__(self, path, factory, create)
 830         if not os.path.exists(self._path):
 831             if create:
 832                 os.mkdir(self._path, 0o700)
 833                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 834                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600))
 835             else:
 836                 raise NoSuchMailboxError(self._path)
 837         self._locked = False
 838
 839     def add(self, message):
 840         """Add message and return assigned key."""
 841         keys = self.keys()
 842         if len(keys) == 0:
 843             new_key = 1
 844         else:
 845             new_key = max(keys) + 1
 846         new_path = os.path.join(self._path, str(new_key))
 847         f = _create_carefully(new_path)
 848         try:
 849             if self._locked:
 850                 _lock_file(f)
 851             try:
 852                 self._dump_message(message, f)
 853                 if isinstance(message, MHMessage):
 854                     self._dump_sequences(message, new_key)
 855             finally:
 856                 if self._locked:
 857                     _unlock_file(f)
 858         finally:
 859             _sync_close(f)
 860         return new_key
 861
 862     def remove(self, key):
 863         """Remove the keyed message; raise KeyError if it doesn't exist."""
 864         path = os.path.join(self._path, str(key))
 865         try:
 866             f = open(path, 'rb+')
 867         except IOError as e:
 868             if e.errno == errno.ENOENT:
 869                 raise KeyError('No message with key: %s' % key)
 870             else:
 871                 raise
 872         try:
 873             if self._locked:
 874                 _lock_file(f)
 875             try:
 876                 f.close()
 877                 os.remove(os.path.join(self._path, str(key)))
 878             finally:
 879                 if self._locked:
 880                     _unlock_file(f)
 881         finally:
 882             f.close()
 883
 884     def __setitem__(self, key, message):
 885         """Replace the keyed message; raise KeyError if it doesn't exist."""
 886         path = os.path.join(self._path, str(key))
 887         try:
 888             f = open(path, 'r+', newline='')
 889         except IOError as e:
 890             if e.errno == errno.ENOENT:
 891                 raise KeyError('No message with key: %s' % key)
 892             else:
 893                 raise
 894         try:
 895             if self._locked:
 896                 _lock_file(f)
 897             try:
 898                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 899                 self._dump_message(message, f)
 900                 if isinstance(message, MHMessage):
 901                     self._dump_sequences(message, key)
 902             finally:
 903                 if self._locked:
 904                     _unlock_file(f)
 905         finally:
 906             _sync_close(f)
 907
 908     def get_message(self, key):
 909         """Return a Message representation or raise a KeyError."""
 910         try:
 911             if self._locked:
 912                 f = open(os.path.join(self._path, str(key)), 'r+', newline='')
 913             else:
 914                 f = open(os.path.join(self._path, str(key)), 'r', newline='')
 915         except IOError as e:
 916             if e.errno == errno.ENOENT:
 917                 raise KeyError('No message with key: %s' % key)
 918             else:
 919                 raise
 920         try:
 921             if self._locked:
 922                 _lock_file(f)
 923             try:
 924                 msg = MHMessage(f)
 925             finally:
 926                 if self._locked:
 927                     _unlock_file(f)
 928         finally:
 929             f.close()
 930         for name, key_list in self.get_sequences().items():
 931             if key in key_list:
 932                 msg.add_sequence(name)
 933         return msg
 934
 935     def get_string(self, key):
 936         """Return a string representation or raise a KeyError."""
 937         try:
 938             if self._locked:
 939                 f = open(os.path.join(self._path, str(key)), 'r+', newline='')
 940             else:
 941                 f = open(os.path.join(self._path, str(key)), 'r', newline='')
 942         except IOError as e:
 943             if e.errno == errno.ENOENT:
 944                 raise KeyError('No message with key: %s' % key)
 945             else:
 946                 raise
 947         try:
 948             if self._locked:
 949                 _lock_file(f)
 950             try:
 951                 return f.read()
 952             finally:
 953                 if self._locked:
 954                     _unlock_file(f)
 955         finally:
 956             f.close()
 957
 958     def get_file(self, key):
 959         """Return a file-like representation or raise a KeyError."""
 960         try:
 961             f = open(os.path.join(self._path, str(key)), 'r', newline='')
 962         except IOError as e:
 963             if e.errno == errno.ENOENT:
 964                 raise KeyError('No message with key: %s' % key)
 965             else:
 966                 raise
 967         return _ProxyFile(f)
 968
 969     def iterkeys(self):
 970         """Return an iterator over keys."""
 971         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 972                                       if entry.isdigit()))
 973
 974     def __contains__(self, key):
 975         """Return True if the keyed message exists, False otherwise."""
 976         return os.path.exists(os.path.join(self._path, str(key)))
 977
 978     def __len__(self):
 979         """Return a count of messages in the mailbox."""
 980         return len(list(self.keys()))
 981
 982     def lock(self):
 983         """Lock the mailbox."""
 984         if not self._locked:
 985             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
 986             _lock_file(self._file)
 987             self._locked = True
 988
 989     def unlock(self):
 990         """Unlock the mailbox if it is locked."""
 991         if self._locked:
 992             _unlock_file(self._file)
 993             _sync_close(self._file)
 994             del self._file
 995             self._locked = False
 996
 997     def flush(self):
 998         """Write any pending changes to the disk."""
 999         return
1000
1001     def close(self):
1002         """Flush and close the mailbox."""
1003         if self._locked:
1004             self.unlock()
1005
1006     def list_folders(self):
1007         """Return a list of folder names."""
1008         result = []
1009         for entry in os.listdir(self._path):
1010             if os.path.isdir(os.path.join(self._path, entry)):
1011                 result.append(entry)
1012         return result
1013
1014     def get_folder(self, folder):
1015         """Return an MH instance for the named folder."""
1016         return MH(os.path.join(self._path, folder),
1017                   factory=self._factory, create=False)
1018
1019     def add_folder(self, folder):
1020         """Create a folder and return an MH instance representing it."""
1021         return MH(os.path.join(self._path, folder),
1022                   factory=self._factory)
1023
1024     def remove_folder(self, folder):
1025         """Delete the named folder, which must be empty."""
1026         path = os.path.join(self._path, folder)
1027         entries = os.listdir(path)
1028         if entries == ['.mh_sequences']:
1029             os.remove(os.path.join(path, '.mh_sequences'))
1030         elif entries == []:
1031             pass
1032         else:
1033             raise NotEmptyError('Folder not empty: %s' % self._path)
1034         os.rmdir(path)
1035
1036     def get_sequences(self):
1037         """Return a name-to-key-list dictionary to define each sequence."""
1038         results = {}
1039         f = open(os.path.join(self._path, '.mh_sequences'), 'r', newline='')
1040         try:
1041             all_keys = set(self.keys())
1042             for line in f:
1043                 try:
1044                     name, contents = line.split(':')
1045                     keys = set()
1046                     for spec in contents.split():
1047                         if spec.isdigit():
1048                             keys.add(int(spec))
1049                         else:
1050                             start, stop = (int(x) for x in spec.split('-'))
1051                             keys.update(range(start, stop + 1))
1052                     results[name] = [key for key in sorted(keys) \
1053                                          if key in all_keys]
1054                     if len(results[name]) == 0:
1055                         del results[name]
1056                 except ValueError:
1057                     raise FormatError('Invalid sequence specification: %s' %
1058                                       line.rstrip())
1059         finally:
1060             f.close()
1061         return results
1062
1063     def set_sequences(self, sequences):
1064         """Set sequences using the given name-to-key-list dictionary."""
1065         f = open(os.path.join(self._path, '.mh_sequences'), 'r+', newline='')
1066         try:
1067             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1068             for name, keys in sequences.items():
1069                 if len(keys) == 0:
1070                     continue
1071                 f.write('%s:' % name)
1072                 prev = None
1073                 completing = False
1074                 for key in sorted(set(keys)):
1075                     if key - 1 == prev:
1076                         if not completing:
1077                             completing = True
1078                             f.write('-')
1079                     elif completing:
1080                         completing = False
1081                         f.write('%s %s' % (prev, key))
1082                     else:
1083                         f.write(' %s' % key)
1084                     prev = key
1085                 if completing:
1086                     f.write(str(prev) + '\n')
1087                 else:
1088                     f.write('\n')
1089         finally:
1090             _sync_close(f)
1091
1092     def pack(self):
1093         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1094         sequences = self.get_sequences()
1095         prev = 0
1096         changes = []
1097         for key in self.keys():
1098             if key - 1 != prev:
1099                 changes.append((key, prev + 1))
1100                 if hasattr(os, 'link'):
1101                     os.link(os.path.join(self._path, str(key)),
1102                             os.path.join(self._path, str(prev + 1)))
1103                     os.unlink(os.path.join(self._path, str(key)))
1104                 else:
1105                     os.rename(os.path.join(self._path, str(key)),
1106                               os.path.join(self._path, str(prev + 1)))
1107             prev += 1
1108         self._next_key = prev + 1
1109         if len(changes) == 0:
1110             return
1111         for name, key_list in sequences.items():
1112             for old, new in changes:
1113                 if old in key_list:
1114                     key_list[key_list.index(old)] = new
1115         self.set_sequences(sequences)
1116
1117     def _dump_sequences(self, message, key):
1118         """Inspect a new MHMessage and update sequences appropriately."""
1119         pending_sequences = message.get_sequences()
1120         all_sequences = self.get_sequences()
1121         for name, key_list in all_sequences.items():
1122             if name in pending_sequences:
1123                 key_list.append(key)
1124             elif key in key_list:
1125                 del key_list[key_list.index(key)]
1126         for sequence in pending_sequences:
1127             if sequence not in all_sequences:
1128                 all_sequences[sequence] = [key]
1129         self.set_sequences(all_sequences)
1130
1131
1132 class Babyl(_singlefileMailbox):
1133     """An Rmail-style Babyl mailbox."""
1134
1135     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1136                                  'forwarded', 'edited', 'resent'))
1137
1138     def __init__(self, path, factory=None, create=True):
1139         """Initialize a Babyl mailbox."""
1140         _singlefileMailbox.__init__(self, path, factory, create)
1141         self._labels = {}
1142
1143     def add(self, message):
1144         """Add message and return assigned key."""
1145         key = _singlefileMailbox.add(self, message)
1146         if isinstance(message, BabylMessage):
1147             self._labels[key] = message.get_labels()
1148         return key
1149
1150     def remove(self, key):
1151         """Remove the keyed message; raise KeyError if it doesn't exist."""
1152         _singlefileMailbox.remove(self, key)
1153         if key in self._labels:
1154             del self._labels[key]
1155
1156     def __setitem__(self, key, message):
1157         """Replace the keyed message; raise KeyError if it doesn't exist."""
1158         _singlefileMailbox.__setitem__(self, key, message)
1159         if isinstance(message, BabylMessage):
1160             self._labels[key] = message.get_labels()
1161
1162     def get_message(self, key):
1163         """Return a Message representation or raise a KeyError."""
1164         start, stop = self._lookup(key)
1165         self._file.seek(start)
1166         self._file.readline()   # Skip '1,' line specifying labels.
1167         original_headers = io.StringIO()
1168         while True:
1169             line = self._file.readline()
1170             if line == '*** EOOH ***' + os.linesep or not line:
1171                 break
1172             original_headers.write(line.replace(os.linesep, '\n'))
1173         visible_headers = io.StringIO()
1174         while True:
1175             line = self._file.readline()
1176             if line == os.linesep or not line:
1177                 break
1178             visible_headers.write(line.replace(os.linesep, '\n'))
1179         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1180                                                                  '\n')
1181         msg = BabylMessage(original_headers.getvalue() + body)
1182         msg.set_visible(visible_headers.getvalue())
1183         if key in self._labels:
1184             msg.set_labels(self._labels[key])
1185         return msg
1186
1187     def get_string(self, key):
1188         """Return a string representation or raise a KeyError."""
1189         start, stop = self._lookup(key)
1190         self._file.seek(start)
1191         self._file.readline()   # Skip '1,' line specifying labels.
1192         original_headers = io.StringIO()
1193         while True:
1194             line = self._file.readline()
1195             if line == '*** EOOH ***' + os.linesep or not line:
1196                 break
1197             original_headers.write(line.replace(os.linesep, '\n'))
1198         while True:
1199             line = self._file.readline()
1200             if line == os.linesep or not line:
1201                 break
1202         return original_headers.getvalue() + \
1203                self._file.read(stop - self._file.tell()).replace(os.linesep,
1204                                                                  '\n')
1205
1206     def get_file(self, key):
1207         """Return a file-like representation or raise a KeyError."""
1208         return io.StringIO(self.get_string(key).replace('\n',
1209                                                               os.linesep))
1210
1211     def get_labels(self):
1212         """Return a list of user-defined labels in the mailbox."""
1213         self._lookup()
1214         labels = set()
1215         for label_list in self._labels.values():
1216             labels.update(label_list)
1217         labels.difference_update(self._special_labels)
1218         return list(labels)
1219
1220     def _generate_toc(self):
1221         """Generate key-to-(start, stop) table of contents."""
1222         starts, stops = [], []
1223         self._file.seek(0)
1224         next_pos = 0
1225         label_lists = []
1226         while True:
1227             line_pos = next_pos
1228             line = self._file.readline()
1229             next_pos = self._file.tell()
1230             if line == '\037\014' + os.linesep:
1231                 if len(stops) < len(starts):
1232                     stops.append(line_pos - len(os.linesep))
1233                 starts.append(next_pos)
1234                 labels = [label.strip() for label
1235                                         in self._file.readline()[1:].split(',')
1236                                         if label.strip()]
1237                 label_lists.append(labels)
1238             elif line == '\037' or line == '\037' + os.linesep:
1239                 if len(stops) < len(starts):
1240                     stops.append(line_pos - len(os.linesep))
1241             elif not line:
1242                 stops.append(line_pos - len(os.linesep))
1243                 break
1244         self._toc = dict(enumerate(zip(starts, stops)))
1245         self._labels = dict(enumerate(label_lists))
1246         self._next_key = len(self._toc)
1247         self._file.seek(0, 2)
1248         self._file_length = self._file.tell()
1249
1250     def _pre_mailbox_hook(self, f):
1251         """Called before writing the mailbox to file f."""
1252         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1253                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1254                  os.linesep))
1255
1256     def _pre_message_hook(self, f):
1257         """Called before writing each message to file f."""
1258         f.write('\014' + os.linesep)
1259
1260     def _post_message_hook(self, f):
1261         """Called after writing each message to file f."""
1262         f.write(os.linesep + '\037')
1263
1264     def _install_message(self, message):
1265         """Write message contents and return (start, stop)."""
1266         start = self._file.tell()
1267         if isinstance(message, BabylMessage):
1268             special_labels = []
1269             labels = []
1270             for label in message.get_labels():
1271                 if label in self._special_labels:
1272                     special_labels.append(label)
1273                 else:
1274                     labels.append(label)
1275             self._file.write('1')
1276             for label in special_labels:
1277                 self._file.write(', ' + label)
1278             self._file.write(',,')
1279             for label in labels:
1280                 self._file.write(' ' + label + ',')
1281             self._file.write(os.linesep)
1282         else:
1283             self._file.write('1,,' + os.linesep)
1284         if isinstance(message, email.message.Message):
1285             orig_buffer = io.StringIO()
1286             orig_generator = email.generator.Generator(orig_buffer, False, 0)
1287             orig_generator.flatten(message)
1288             orig_buffer.seek(0)
1289             while True:
1290                 line = orig_buffer.readline()
1291                 self._file.write(line.replace('\n', os.linesep))
1292                 if line == '\n' or not line:
1293                     break
1294             self._file.write('*** EOOH ***' + os.linesep)
1295             if isinstance(message, BabylMessage):
1296                 vis_buffer = io.StringIO()
1297                 vis_generator = email.generator.Generator(vis_buffer, False, 0)
1298                 vis_generator.flatten(message.get_visible())
1299                 while True:
1300                     line = vis_buffer.readline()
1301                     self._file.write(line.replace('\n', os.linesep))
1302                     if line == '\n' or not line:
1303                         break
1304             else:
1305                 orig_buffer.seek(0)
1306                 while True:
1307                     line = orig_buffer.readline()
1308                     self._file.write(line.replace('\n', os.linesep))
1309                     if line == '\n' or not line:
1310                         break
1311             while True:
1312                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1313                 if not buffer:
1314                     break
1315                 self._file.write(buffer.replace('\n', os.linesep))
1316         elif isinstance(message, str):
1317             body_start = message.find('\n\n') + 2
1318             if body_start - 2 != -1:
1319                 self._file.write(message[:body_start].replace('\n',
1320                                                               os.linesep))
1321                 self._file.write('*** EOOH ***' + os.linesep)
1322                 self._file.write(message[:body_start].replace('\n',
1323                                                               os.linesep))
1324                 self._file.write(message[body_start:].replace('\n',
1325                                                               os.linesep))
1326             else:
1327                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1328                 self._file.write(message.replace('\n', os.linesep))
1329         elif hasattr(message, 'readline'):
1330             original_pos = message.tell()
1331             first_pass = True
1332             while True:
1333                 line = message.readline()
1334                 self._file.write(line.replace('\n', os.linesep))
1335                 if line == '\n' or not line:
1336                     self._file.write('*** EOOH ***' + os.linesep)
1337                     if first_pass:
1338                         first_pass = False
1339                         message.seek(original_pos)
1340                     else:
1341                         break
1342             while True:
1343                 buffer = message.read(4096)     # Buffer size is arbitrary.
1344                 if not buffer:
1345                     break
1346                 self._file.write(buffer.replace('\n', os.linesep))
1347         else:
1348             raise TypeError('Invalid message type: %s' % type(message))
1349         stop = self._file.tell()
1350         return (start, stop)
1351
1352
1353 class Message(email.message.Message):
1354     """Message with mailbox-format-specific properties."""
1355
1356     def __init__(self, message=None):
1357         """Initialize a Message instance."""
1358         if isinstance(message, email.message.Message):
1359             self._become_message(copy.deepcopy(message))
1360             if isinstance(message, Message):
1361                 message._explain_to(self)
1362         elif isinstance(message, str):
1363             self._become_message(email.message_from_string(message))
1364         elif hasattr(message, "read"):
1365             self._become_message(email.message_from_file(message))
1366         elif message is None:
1367             email.message.Message.__init__(self)
1368         else:
1369             raise TypeError('Invalid message type: %s' % type(message))
1370
1371     def _become_message(self, message):
1372         """Assume the non-format-specific state of message."""
1373         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1374                      'preamble', 'epilogue', 'defects', '_default_type'):
1375             self.__dict__[name] = message.__dict__[name]
1376
1377     def _explain_to(self, message):
1378         """Copy format-specific state to message insofar as possible."""
1379         if isinstance(message, Message):
1380             return  # There's nothing format-specific to explain.
1381         else:
1382             raise TypeError('Cannot convert to specified type')
1383
1384
1385 class MaildirMessage(Message):
1386     """Message with Maildir-specific properties."""
1387
1388     def __init__(self, message=None):
1389         """Initialize a MaildirMessage instance."""
1390         self._subdir = 'new'
1391         self._info = ''
1392         self._date = time.time()
1393         Message.__init__(self, message)
1394
1395     def get_subdir(self):
1396         """Return 'new' or 'cur'."""
1397         return self._subdir
1398
1399     def set_subdir(self, subdir):
1400         """Set subdir to 'new' or 'cur'."""
1401         if subdir == 'new' or subdir == 'cur':
1402             self._subdir = subdir
1403         else:
1404             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1405
1406     def get_flags(self):
1407         """Return as a string the flags that are set."""
1408         if self._info.startswith('2,'):
1409             return self._info[2:]
1410         else:
1411             return ''
1412
1413     def set_flags(self, flags):
1414         """Set the given flags and unset all others."""
1415         self._info = '2,' + ''.join(sorted(flags))
1416
1417     def add_flag(self, flag):
1418         """Set the given flag(s) without changing others."""
1419         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1420
1421     def remove_flag(self, flag):
1422         """Unset the given string flag(s) without changing others."""
1423         if self.get_flags():
1424             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1425
1426     def get_date(self):
1427         """Return delivery date of message, in seconds since the epoch."""
1428         return self._date
1429
1430     def set_date(self, date):
1431         """Set delivery date of message, in seconds since the epoch."""
1432         try:
1433             self._date = float(date)
1434         except ValueError:
1435             raise TypeError("can't convert to float: %s" % date)
1436
1437     def get_info(self):
1438         """Get the message's "info" as a string."""
1439         return self._info
1440
1441     def set_info(self, info):
1442         """Set the message's "info" string."""
1443         if isinstance(info, str):
1444             self._info = info
1445         else:
1446             raise TypeError('info must be a string: %s' % type(info))
1447
1448     def _explain_to(self, message):
1449         """Copy Maildir-specific state to message insofar as possible."""
1450         if isinstance(message, MaildirMessage):
1451             message.set_flags(self.get_flags())
1452             message.set_subdir(self.get_subdir())
1453             message.set_date(self.get_date())
1454         elif isinstance(message, _mboxMMDFMessage):
1455             flags = set(self.get_flags())
1456             if 'S' in flags:
1457                 message.add_flag('R')
1458             if self.get_subdir() == 'cur':
1459                 message.add_flag('O')
1460             if 'T' in flags:
1461                 message.add_flag('D')
1462             if 'F' in flags:
1463                 message.add_flag('F')
1464             if 'R' in flags:
1465                 message.add_flag('A')
1466             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1467         elif isinstance(message, MHMessage):
1468             flags = set(self.get_flags())
1469             if 'S' not in flags:
1470                 message.add_sequence('unseen')
1471             if 'R' in flags:
1472                 message.add_sequence('replied')
1473             if 'F' in flags:
1474                 message.add_sequence('flagged')
1475         elif isinstance(message, BabylMessage):
1476             flags = set(self.get_flags())
1477             if 'S' not in flags:
1478                 message.add_label('unseen')
1479             if 'T' in flags:
1480                 message.add_label('deleted')
1481             if 'R' in flags:
1482                 message.add_label('answered')
1483             if 'P' in flags:
1484                 message.add_label('forwarded')
1485         elif isinstance(message, Message):
1486             pass
1487         else:
1488             raise TypeError('Cannot convert to specified type: %s' %
1489                             type(message))
1490
1491
1492 class _mboxMMDFMessage(Message):
1493     """Message with mbox- or MMDF-specific properties."""
1494
1495     def __init__(self, message=None):
1496         """Initialize an mboxMMDFMessage instance."""
1497         self.set_from('MAILER-DAEMON', True)
1498         if isinstance(message, email.message.Message):
1499             unixfrom = message.get_unixfrom()
1500             if unixfrom is not None and unixfrom.startswith('From '):
1501                 self.set_from(unixfrom[5:])
1502         Message.__init__(self, message)
1503
1504     def get_from(self):
1505         """Return contents of "From " line."""
1506         return self._from
1507
1508     def set_from(self, from_, time_=None):
1509         """Set "From " line, formatting and appending time_ if specified."""
1510         if time_ is not None:
1511             if time_ is True:
1512                 time_ = time.gmtime()
1513             from_ += ' ' + time.asctime(time_)
1514         self._from = from_
1515
1516     def get_flags(self):
1517         """Return as a string the flags that are set."""
1518         return self.get('Status', '') + self.get('X-Status', '')
1519
1520     def set_flags(self, flags):
1521         """Set the given flags and unset all others."""
1522         flags = set(flags)
1523         status_flags, xstatus_flags = '', ''
1524         for flag in ('R', 'O'):
1525             if flag in flags:
1526                 status_flags += flag
1527                 flags.remove(flag)
1528         for flag in ('D', 'F', 'A'):
1529             if flag in flags:
1530                 xstatus_flags += flag
1531                 flags.remove(flag)
1532         xstatus_flags += ''.join(sorted(flags))
1533         try:
1534             self.replace_header('Status', status_flags)
1535         except KeyError:
1536             self.add_header('Status', status_flags)
1537         try:
1538             self.replace_header('X-Status', xstatus_flags)
1539         except KeyError:
1540             self.add_header('X-Status', xstatus_flags)
1541
1542     def add_flag(self, flag):
1543         """Set the given flag(s) without changing others."""
1544         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1545
1546     def remove_flag(self, flag):
1547         """Unset the given string flag(s) without changing others."""
1548         if 'Status' in self or 'X-Status' in self:
1549             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1550
1551     def _explain_to(self, message):
1552         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1553         if isinstance(message, MaildirMessage):
1554             flags = set(self.get_flags())
1555             if 'O' in flags:
1556                 message.set_subdir('cur')
1557             if 'F' in flags:
1558                 message.add_flag('F')
1559             if 'A' in flags:
1560                 message.add_flag('R')
1561             if 'R' in flags:
1562                 message.add_flag('S')
1563             if 'D' in flags:
1564                 message.add_flag('T')
1565             del message['status']
1566             del message['x-status']
1567             maybe_date = ' '.join(self.get_from().split()[-5:])
1568             try:
1569                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1570                                                       '%a %b %d %H:%M:%S %Y')))
1571             except (ValueError, OverflowError):
1572                 pass
1573         elif isinstance(message, _mboxMMDFMessage):
1574             message.set_flags(self.get_flags())
1575             message.set_from(self.get_from())
1576         elif isinstance(message, MHMessage):
1577             flags = set(self.get_flags())
1578             if 'R' not in flags:
1579                 message.add_sequence('unseen')
1580             if 'A' in flags:
1581                 message.add_sequence('replied')
1582             if 'F' in flags:
1583                 message.add_sequence('flagged')
1584             del message['status']
1585             del message['x-status']
1586         elif isinstance(message, BabylMessage):
1587             flags = set(self.get_flags())
1588             if 'R' not in flags:
1589                 message.add_label('unseen')
1590             if 'D' in flags:
1591                 message.add_label('deleted')
1592             if 'A' in flags:
1593                 message.add_label('answered')
1594             del message['status']
1595             del message['x-status']
1596         elif isinstance(message, Message):
1597             pass
1598         else:
1599             raise TypeError('Cannot convert to specified type: %s' %
1600                             type(message))
1601
1602
1603 class mboxMessage(_mboxMMDFMessage):
1604     """Message with mbox-specific properties."""
1605
1606
1607 class MHMessage(Message):
1608     """Message with MH-specific properties."""
1609
1610     def __init__(self, message=None):
1611         """Initialize an MHMessage instance."""
1612         self._sequences = []
1613         Message.__init__(self, message)
1614
1615     def get_sequences(self):
1616         """Return a list of sequences that include the message."""
1617         return self._sequences[:]
1618
1619     def set_sequences(self, sequences):
1620         """Set the list of sequences that include the message."""
1621         self._sequences = list(sequences)
1622
1623     def add_sequence(self, sequence):
1624         """Add sequence to list of sequences including the message."""
1625         if isinstance(sequence, str):
1626             if not sequence in self._sequences:
1627                 self._sequences.append(sequence)
1628         else:
1629             raise TypeError('sequence must be a string: %s' % type(sequence))
1630
1631     def remove_sequence(self, sequence):
1632         """Remove sequence from the list of sequences including the message."""
1633         try:
1634             self._sequences.remove(sequence)
1635         except ValueError:
1636             pass
1637
1638     def _explain_to(self, message):
1639         """Copy MH-specific state to message insofar as possible."""
1640         if isinstance(message, MaildirMessage):
1641             sequences = set(self.get_sequences())
1642             if 'unseen' in sequences:
1643                 message.set_subdir('cur')
1644             else:
1645                 message.set_subdir('cur')
1646                 message.add_flag('S')
1647             if 'flagged' in sequences:
1648                 message.add_flag('F')
1649             if 'replied' in sequences:
1650                 message.add_flag('R')
1651         elif isinstance(message, _mboxMMDFMessage):
1652             sequences = set(self.get_sequences())
1653             if 'unseen' not in sequences:
1654                 message.add_flag('RO')
1655             else:
1656                 message.add_flag('O')
1657             if 'flagged' in sequences:
1658                 message.add_flag('F')
1659             if 'replied' in sequences:
1660                 message.add_flag('A')
1661         elif isinstance(message, MHMessage):
1662             for sequence in self.get_sequences():
1663                 message.add_sequence(sequence)
1664         elif isinstance(message, BabylMessage):
1665             sequences = set(self.get_sequences())
1666             if 'unseen' in sequences:
1667                 message.add_label('unseen')
1668             if 'replied' in sequences:
1669                 message.add_label('answered')
1670         elif isinstance(message, Message):
1671             pass
1672         else:
1673             raise TypeError('Cannot convert to specified type: %s' %
1674                             type(message))
1675
1676
1677 class BabylMessage(Message):
1678     """Message with Babyl-specific properties."""
1679
1680     def __init__(self, message=None):
1681         """Initialize an BabylMessage instance."""
1682         self._labels = []
1683         self._visible = Message()
1684         Message.__init__(self, message)
1685
1686     def get_labels(self):
1687         """Return a list of labels on the message."""
1688         return self._labels[:]
1689
1690     def set_labels(self, labels):
1691         """Set the list of labels on the message."""
1692         self._labels = list(labels)
1693
1694     def add_label(self, label):
1695         """Add label to list of labels on the message."""
1696         if isinstance(label, str):
1697             if label not in self._labels:
1698                 self._labels.append(label)
1699         else:
1700             raise TypeError('label must be a string: %s' % type(label))
1701
1702     def remove_label(self, label):
1703         """Remove label from the list of labels on the message."""
1704         try:
1705             self._labels.remove(label)
1706         except ValueError:
1707             pass
1708
1709     def get_visible(self):
1710         """Return a Message representation of visible headers."""
1711         return Message(self._visible)
1712
1713     def set_visible(self, visible):
1714         """Set the Message representation of visible headers."""
1715         self._visible = Message(visible)
1716
1717     def update_visible(self):
1718         """Update and/or sensibly generate a set of visible headers."""
1719         for header in self._visible.keys():
1720             if header in self:
1721                 self._visible.replace_header(header, self[header])
1722             else:
1723                 del self._visible[header]
1724         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1725             if header in self and header not in self._visible:
1726                 self._visible[header] = self[header]
1727
1728     def _explain_to(self, message):
1729         """Copy Babyl-specific state to message insofar as possible."""
1730         if isinstance(message, MaildirMessage):
1731             labels = set(self.get_labels())
1732             if 'unseen' in labels:
1733                 message.set_subdir('cur')
1734             else:
1735                 message.set_subdir('cur')
1736                 message.add_flag('S')
1737             if 'forwarded' in labels or 'resent' in labels:
1738                 message.add_flag('P')
1739             if 'answered' in labels:
1740                 message.add_flag('R')
1741             if 'deleted' in labels:
1742                 message.add_flag('T')
1743         elif isinstance(message, _mboxMMDFMessage):
1744             labels = set(self.get_labels())
1745             if 'unseen' not in labels:
1746                 message.add_flag('RO')
1747             else:
1748                 message.add_flag('O')
1749             if 'deleted' in labels:
1750                 message.add_flag('D')
1751             if 'answered' in labels:
1752                 message.add_flag('A')
1753         elif isinstance(message, MHMessage):
1754             labels = set(self.get_labels())
1755             if 'unseen' in labels:
1756                 message.add_sequence('unseen')
1757             if 'answered' in labels:
1758                 message.add_sequence('replied')
1759         elif isinstance(message, BabylMessage):
1760             message.set_visible(self.get_visible())
1761             for label in self.get_labels():
1762                 message.add_label(label)
1763         elif isinstance(message, Message):
1764             pass
1765         else:
1766             raise TypeError('Cannot convert to specified type: %s' %
1767                             type(message))
1768
1769
1770 class MMDFMessage(_mboxMMDFMessage):
1771     """Message with MMDF-specific properties."""
1772
1773
1774 class _ProxyFile:
1775     """A read-only wrapper of a file."""
1776
1777     def __init__(self, f, pos=None):
1778         """Initialize a _ProxyFile."""
1779         self._file = f
1780         if pos is None:
1781             self._pos = f.tell()
1782         else:
1783             self._pos = pos
1784
1785     def read(self, size=None):
1786         """Read bytes."""
1787         return self._read(size, self._file.read)
1788
1789     def readline(self, size=None):
1790         """Read a line."""
1791         return self._read(size, self._file.readline)
1792
1793     def readlines(self, sizehint=None):
1794         """Read multiple lines."""
1795         result = []
1796         for line in self:
1797             result.append(line)
1798             if sizehint is not None:
1799                 sizehint -= len(line)
1800                 if sizehint <= 0:
1801                     break
1802         return result
1803
1804     def __iter__(self):
1805         """Iterate over lines."""
1806         while True:
1807             line = self.readline()
1808             if not line:
1809                 raise StopIteration
1810             yield line
1811
1812     def tell(self):
1813         """Return the position."""
1814         return self._pos
1815
1816     def seek(self, offset, whence=0):
1817         """Change position."""
1818         if whence == 1:
1819             self._file.seek(self._pos)
1820         self._file.seek(offset, whence)
1821         self._pos = self._file.tell()
1822
1823     def close(self):
1824         """Close the file."""
1825         del self._file
1826
1827     def _read(self, size, read_method):
1828         """Read size bytes using read_method."""
1829         if size is None:
1830             size = -1
1831         self._file.seek(self._pos)
1832         result = read_method(size)
1833         self._pos = self._file.tell()
1834         return result
1835
1836
1837 class _PartialFile(_ProxyFile):
1838     """A read-only wrapper of part of a file."""
1839
1840     def __init__(self, f, start=None, stop=None):
1841         """Initialize a _PartialFile."""
1842         _ProxyFile.__init__(self, f, start)
1843         self._start = start
1844         self._stop = stop
1845
1846     def tell(self):
1847         """Return the position with respect to start."""
1848         return _ProxyFile.tell(self) - self._start
1849
1850     def seek(self, offset, whence=0):
1851         """Change position, possibly with respect to start or stop."""
1852         if whence == 0:
1853             self._pos = self._start
1854             whence = 1
1855         elif whence == 2:
1856             self._pos = self._stop
1857             whence = 1
1858         _ProxyFile.seek(self, offset, whence)
1859
1860     def _read(self, size, read_method):
1861         """Read size bytes using read_method, honoring start and stop."""
1862         remaining = self._stop - self._pos
1863         if remaining <= 0:
1864             return ''
1865         if size is None or size < 0 or size > remaining:
1866             size = remaining
1867         return _ProxyFile._read(self, size, read_method)
1868
1869
1870 def _lock_file(f, dotlock=True):
1871     """Lock file f using lockf and dot locking."""
1872     dotlock_done = False
1873     try:
1874         if fcntl:
1875             try:
1876                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1877             except IOError as e:
1878                 if e.errno in (errno.EAGAIN, errno.EACCES):
1879                     raise ExternalClashError('lockf: lock unavailable: %s' %
1880                                              f.name)
1881                 else:
1882                     raise
1883         if dotlock:
1884             try:
1885                 pre_lock = _create_temporary(f.name + '.lock')
1886                 pre_lock.close()
1887             except IOError as e:
1888                 if e.errno == errno.EACCES:
1889                     return  # Without write access, just skip dotlocking.
1890                 else:
1891                     raise
1892             try:
1893                 if hasattr(os, 'link'):
1894                     os.link(pre_lock.name, f.name + '.lock')
1895                     dotlock_done = True
1896                     os.unlink(pre_lock.name)
1897                 else:
1898                     os.rename(pre_lock.name, f.name + '.lock')
1899                     dotlock_done = True
1900             except OSError as e:
1901                 if e.errno == errno.EEXIST or \
1902                   (os.name == 'os2' and e.errno == errno.EACCES):
1903                     os.remove(pre_lock.name)
1904                     raise ExternalClashError('dot lock unavailable: %s' %
1905                                              f.name)
1906                 else:
1907                     raise
1908     except:
1909         if fcntl:
1910             fcntl.lockf(f, fcntl.LOCK_UN)
1911         if dotlock_done:
1912             os.remove(f.name + '.lock')
1913         raise
1914
1915 def _unlock_file(f):
1916     """Unlock file f using lockf and dot locking."""
1917     if fcntl:
1918         fcntl.lockf(f, fcntl.LOCK_UN)
1919     if os.path.exists(f.name + '.lock'):
1920         os.remove(f.name + '.lock')
1921
1922 def _create_carefully(path):
1923     """Create a file if it doesn't exist and open for reading and writing."""
1924     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666)
1925     try:
1926         return open(path, 'r+', newline='')
1927     finally:
1928         os.close(fd)
1929
1930 def _create_temporary(path):
1931     """Create a temp file based on path and open for reading and writing."""
1932     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1933                                               socket.gethostname(),
1934                                               os.getpid()))
1935
1936 def _sync_flush(f):
1937     """Ensure changes to file f are physically on disk."""
1938     f.flush()
1939     if hasattr(os, 'fsync'):
1940         os.fsync(f.fileno())
1941
1942 def _sync_close(f):
1943     """Close file f, ensuring all changes are physically on disk."""
1944     _sync_flush(f)
1945     f.close()
1946
1947
1948 class Error(Exception):
1949     """Raised for module-specific errors."""
1950
1951 class NoSuchMailboxError(Error):
1952     """The specified mailbox does not exist and won't be created."""
1953
1954 class NotEmptyError(Error):
1955     """The specified mailbox is not empty and deletion was requested."""
1956
1957 class ExternalClashError(Error):
1958     """Another process caused an action to fail."""
1959
1960 class FormatError(Error):
1961     """A file appears to have an invalid format."""