Lib/shelve.py

   1 """Manage shelves of pickled objects.
   2
   3 A "shelf" is a persistent, dictionary-like object.  The difference
   4 with dbm databases is that the values (not the keys!) in a shelf can
   5 be essentially arbitrary Python objects -- anything that the "pickle"
   6 module can handle.  This includes most class instances, recursive data
   7 types, and objects containing lots of shared sub-objects.  The keys
   8 are ordinary strings.
   9
  10 To summarize the interface (key is a string, data is an arbitrary
  11 object):
  12
  13         import shelve
  14         d = shelve.open(filename) # open, with (g)dbm filename -- no suffix
  15
  16         d[key] = data   # store data at key (overwrites old data if
  17                         # using an existing key)
  18         data = d[key]   # retrieve a COPY of the data at key (raise
  19                         # KeyError if no such key) -- NOTE that this
  20                         # access returns a *copy* of the entry!
  21         del d[key]      # delete data stored at key (raises KeyError
  22                         # if no such key)
  23         flag = d.has_key(key)   # true if the key exists; same as "key in d"
  24         list = d.keys() # a list of all existing keys (slow!)
  25
  26         d.close()       # close it
  27
  28 Dependent on the implementation, closing a persistent dictionary may
  29 or may not be necessary to flush changes to disk.
  30
  31 Normally, d[key] returns a COPY of the entry.  This needs care when
  32 mutable entries are mutated: for example, if d[key] is a list,
  33         d[key].append(anitem)
  34 does NOT modify the entry d[key] itself, as stored in the persistent
  35 mapping -- it only modifies the copy, which is then immediately
  36 discarded, so that the append has NO effect whatsoever.  To append an
  37 item to d[key] in a way that will affect the persistent mapping, use:
  38         data = d[key]
  39         data.append(anitem)
  40         d[key] = data
  41
  42 To avoid the problem with mutable entries, you may pass the keyword
  43 argument writeback=True in the call to shelve.open.  When you use:
  44         d = shelve.open(filename, writeback=True)
  45 then d keeps a cache of all entries you access, and writes them all back
  46 to the persistent mapping when you call d.close().  This ensures that
  47 such usage as d[key].append(anitem) works as intended.
  48
  49 However, using keyword argument writeback=True may consume vast amount
  50 of memory for the cache, and it may make d.close() very slow, if you
  51 access many of d's entries after opening it in this way: d has no way to
  52 check which of the entries you access are mutable and/or which ones you
  53 actually mutate, so it must cache, and write back at close, all of the
  54 entries that you access.  You can call d.sync() to write back all the
  55 entries in the cache, and empty the cache (d.sync() also synchronizes
  56 the persistent dictionary on disk, if feasible).
  57 """
  58
  59 # Try using cPickle and cStringIO if available.
  60
  61 try:
  62     from cPickle import Pickler, Unpickler
  63 except ImportError:
  64     from pickle import Pickler, Unpickler
  65
  66 try:
  67     from cStringIO import StringIO
  68 except ImportError:
  69     from StringIO import StringIO
  70
  71 import UserDict
  72 import warnings
  73
  74 __all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"]
  75
  76 class _ClosedDict(UserDict.DictMixin):
  77     'Marker for a closed dict.  Access attempts raise a ValueError.'
  78
  79     def closed(self, *args):
  80         raise ValueError('invalid operation on closed shelf')
  81     __getitem__ = __setitem__ = __delitem__ = keys = closed
  82
  83     def __repr__(self):
  84         return '<Closed Dictionary>'
  85
  86 class Shelf(UserDict.DictMixin):
  87     """Base class for shelf implementations.
  88
  89     This is initialized with a dictionary-like object.
  90     See the module's __doc__ string for an overview of the interface.
  91     """
  92
  93     def __init__(self, dict, protocol=None, writeback=False):
  94         self.dict = dict
  95         if protocol is None:
  96             protocol = 0
  97         self._protocol = protocol
  98         self.writeback = writeback
  99         self.cache = {}
 100
 101     def keys(self):
 102         return self.dict.keys()
 103
 104     def __len__(self):
 105         return len(self.dict)
 106
 107     def has_key(self, key):
 108         return self.dict.has_key(key)
 109
 110     def __contains__(self, key):
 111         return self.dict.has_key(key)
 112
 113     def get(self, key, default=None):
 114         if self.dict.has_key(key):
 115             return self[key]
 116         return default
 117
 118     def __getitem__(self, key):
 119         try:
 120             value = self.cache[key]
 121         except KeyError:
 122             f = StringIO(self.dict[key])
 123             value = Unpickler(f).load()
 124             if self.writeback:
 125                 self.cache[key] = value
 126         return value
 127
 128     def __setitem__(self, key, value):
 129         if self.writeback:
 130             self.cache[key] = value
 131         f = StringIO()
 132         p = Pickler(f, self._protocol)
 133         p.dump(value)
 134         self.dict[key] = f.getvalue()
 135
 136     def __delitem__(self, key):
 137         del self.dict[key]
 138         try:
 139             del self.cache[key]
 140         except KeyError:
 141             pass
 142
 143     def close(self):
 144         self.sync()
 145         try:
 146             self.dict.close()
 147         except AttributeError:
 148             pass
 149         self.dict = _ClosedDict()
 150
 151     def __del__(self):
 152         if not hasattr(self, 'writeback'):
 153             # __init__ didn't succeed, so don't bother closing
 154             return
 155         self.close()
 156
 157     def sync(self):
 158         if self.writeback and self.cache:
 159             self.writeback = False
 160             for key, entry in self.cache.iteritems():
 161                 self[key] = entry
 162             self.writeback = True
 163             self.cache = {}
 164         if hasattr(self.dict, 'sync'):
 165             self.dict.sync()
 166
 167
 168 class BsdDbShelf(Shelf):
 169     """Shelf implementation using the "BSD" db interface.
 170
 171     This adds methods first(), next(), previous(), last() and
 172     set_location() that have no counterpart in [g]dbm databases.
 173
 174     The actual database must be opened using one of the "bsddb"
 175     modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or
 176     bsddb.rnopen) and passed to the constructor.
 177
 178     See the module's __doc__ string for an overview of the interface.
 179     """
 180
 181     def __init__(self, dict, protocol=None, writeback=False):
 182         Shelf.__init__(self, dict, protocol, writeback)
 183
 184     def set_location(self, key):
 185         (key, value) = self.dict.set_location(key)
 186         f = StringIO(value)
 187         return (key, Unpickler(f).load())
 188
 189     def next(self):
 190         (key, value) = self.dict.next()
 191         f = StringIO(value)
 192         return (key, Unpickler(f).load())
 193
 194     def previous(self):
 195         (key, value) = self.dict.previous()
 196         f = StringIO(value)
 197         return (key, Unpickler(f).load())
 198
 199     def first(self):
 200         (key, value) = self.dict.first()
 201         f = StringIO(value)
 202         return (key, Unpickler(f).load())
 203
 204     def last(self):
 205         (key, value) = self.dict.last()
 206         f = StringIO(value)
 207         return (key, Unpickler(f).load())
 208
 209
 210 class DbfilenameShelf(Shelf):
 211     """Shelf implementation using the "anydbm" generic dbm interface.
 212
 213     This is initialized with the filename for the dbm database.
 214     See the module's __doc__ string for an overview of the interface.
 215     """
 216
 217     def __init__(self, filename, flag='c', protocol=None, writeback=False):
 218         import anydbm
 219         Shelf.__init__(self, anydbm.open(filename, flag), protocol, writeback)
 220
 221
 222 def open(filename, flag='c', protocol=None, writeback=False):
 223     """Open a persistent dictionary for reading and writing.
 224
 225     The filename parameter is the base filename for the underlying
 226     database.  As a side-effect, an extension may be added to the
 227     filename and more than one file may be created.  The optional flag
 228     parameter has the same interpretation as the flag parameter of
 229     anydbm.open(). The optional protocol parameter specifies the
 230     version of the pickle protocol (0, 1, or 2).
 231
 232     See the module's __doc__ string for an overview of the interface.
 233     """
 234
 235     return DbfilenameShelf(filename, flag, protocol, writeback)