Lib/shelve.py

   1 """Manage shelves of pickled objects.
   2
   3 A "shelf" is a persistent, dictionary-like object.  The difference
   4 with dbm databases is that the values (not the keys!) in a shelf can
   5 be essentially arbitrary Python objects -- anything that the "pickle"
   6 module can handle.  This includes most class instances, recursive data
   7 types, and objects containing lots of shared sub-objects.  The keys
   8 are ordinary strings.
   9
  10 To summarize the interface (key is a string, data is an arbitrary
  11 object):
  12
  13         import shelve
  14         d = shelve.open(filename) # open, with (g)dbm filename -- no suffix
  15
  16         d[key] = data   # store data at key (overwrites old data if
  17                         # using an existing key)
  18         data = d[key]   # retrieve a COPY of the data at key (raise
  19                         # KeyError if no such key) -- NOTE that this
  20                         # access returns a *copy* of the entry!
  21         del d[key]      # delete data stored at key (raises KeyError
  22                         # if no such key)
  23         flag = d.has_key(key)   # true if the key exists; same as "key in d"
  24         list = d.keys() # a list of all existing keys (slow!)
  25
  26         d.close()       # close it
  27
  28 Dependent on the implementation, closing a persistent dictionary may
  29 or may not be necessary to flush changes to disk.
  30
  31 Normally, d[key] returns a COPY of the entry.  This needs care when
  32 mutable entries are mutated: for example, if d[key] is a list,
  33         d[key].append(anitem)
  34 does NOT modify the entry d[key] itself, as stored in the persistent
  35 mapping -- it only modifies the copy, which is then immediately
  36 discarded, so that the append has NO effect whatsoever.  To append an
  37 item to d[key] in a way that will affect the persistent mapping, use:
  38         data = d[key]
  39         data.append(anitem)
  40         d[key] = data
  41
  42 To avoid the problem with mutable entries, you may pass the keyword
  43 argument writeback=True in the call to shelve.open.  When you use:
  44         d = shelve.open(filename, writeback=True)
  45 then d keeps a cache of all entries you access, and writes them all back
  46 to the persistent mapping when you call d.close().  This ensures that
  47 such usage as d[key].append(anitem) works as intended.
  48
  49 However, using keyword argument writeback=True may consume vast amount
  50 of memory for the cache, and it may make d.close() very slow, if you
  51 access many of d's entries after opening it in this way: d has no way to
  52 check which of the entries you access are mutable and/or which ones you
  53 actually mutate, so it must cache, and write back at close, all of the
  54 entries that you access.  You can call d.sync() to write back all the
  55 entries in the cache, and empty the cache (d.sync() also synchronizes
  56 the persistent dictionary on disk, if feasible).
  57 """
  58
  59 # Try using cPickle and cStringIO if available.
  60
  61 try:
  62     from cPickle import Pickler, Unpickler
  63 except ImportError:
  64     from pickle import Pickler, Unpickler
  65
  66 try:
  67     from cStringIO import StringIO
  68 except ImportError:
  69     from StringIO import StringIO
  70
  71 import UserDict
  72 import warnings
  73
  74 __all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"]
  75
  76 class Shelf(UserDict.DictMixin):
  77     """Base class for shelf implementations.
  78
  79     This is initialized with a dictionary-like object.
  80     See the module's __doc__ string for an overview of the interface.
  81     """
  82
  83     def __init__(self, dict, protocol=None, writeback=False):
  84         self.dict = dict
  85         if protocol is None:
  86             protocol = 0
  87         self._protocol = protocol
  88         self.writeback = writeback
  89         self.cache = {}
  90
  91     def keys(self):
  92         return self.dict.keys()
  93
  94     def __len__(self):
  95         return len(self.dict)
  96
  97     def has_key(self, key):
  98         return self.dict.has_key(key)
  99
 100     def __contains__(self, key):
 101         return self.dict.has_key(key)
 102
 103     def get(self, key, default=None):
 104         if self.dict.has_key(key):
 105             return self[key]
 106         return default
 107
 108     def __getitem__(self, key):
 109         try:
 110             value = self.cache[key]
 111         except KeyError:
 112             f = StringIO(self.dict[key])
 113             value = Unpickler(f).load()
 114             if self.writeback:
 115                 self.cache[key] = value
 116         return value
 117
 118     def __setitem__(self, key, value):
 119         if self.writeback:
 120             self.cache[key] = value
 121         f = StringIO()
 122         p = Pickler(f, self._protocol)
 123         p.dump(value)
 124         self.dict[key] = f.getvalue()
 125
 126     def __delitem__(self, key):
 127         del self.dict[key]
 128         try:
 129             del self.cache[key]
 130         except KeyError:
 131             pass
 132
 133     def close(self):
 134         self.sync()
 135         try:
 136             self.dict.close()
 137         except AttributeError:
 138             pass
 139         self.dict = 0
 140
 141     def __del__(self):
 142         self.close()
 143
 144     def sync(self):
 145         if self.writeback and self.cache:
 146             self.writeback = False
 147             for key, entry in self.cache.iteritems():
 148                 self[key] = entry
 149             self.writeback = True
 150             self.cache = {}
 151         if hasattr(self.dict, 'sync'):
 152             self.dict.sync()
 153
 154
 155 class BsdDbShelf(Shelf):
 156     """Shelf implementation using the "BSD" db interface.
 157
 158     This adds methods first(), next(), previous(), last() and
 159     set_location() that have no counterpart in [g]dbm databases.
 160
 161     The actual database must be opened using one of the "bsddb"
 162     modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or
 163     bsddb.rnopen) and passed to the constructor.
 164
 165     See the module's __doc__ string for an overview of the interface.
 166     """
 167
 168     def __init__(self, dict, protocol=None, writeback=False):
 169         Shelf.__init__(self, dict, protocol, writeback)
 170
 171     def set_location(self, key):
 172         (key, value) = self.dict.set_location(key)
 173         f = StringIO(value)
 174         return (key, Unpickler(f).load())
 175
 176     def next(self):
 177         (key, value) = self.dict.next()
 178         f = StringIO(value)
 179         return (key, Unpickler(f).load())
 180
 181     def previous(self):
 182         (key, value) = self.dict.previous()
 183         f = StringIO(value)
 184         return (key, Unpickler(f).load())
 185
 186     def first(self):
 187         (key, value) = self.dict.first()
 188         f = StringIO(value)
 189         return (key, Unpickler(f).load())
 190
 191     def last(self):
 192         (key, value) = self.dict.last()
 193         f = StringIO(value)
 194         return (key, Unpickler(f).load())
 195
 196
 197 class DbfilenameShelf(Shelf):
 198     """Shelf implementation using the "anydbm" generic dbm interface.
 199
 200     This is initialized with the filename for the dbm database.
 201     See the module's __doc__ string for an overview of the interface.
 202     """
 203
 204     def __init__(self, filename, flag='c', protocol=None, writeback=False):
 205         import anydbm
 206         Shelf.__init__(self, anydbm.open(filename, flag), protocol, writeback)
 207
 208
 209 def open(filename, flag='c', protocol=None, writeback=False):
 210     """Open a persistent dictionary for reading and writing.
 211
 212     The filename parameter is the base filename for the underlying
 213     database.  As a side-effect, an extension may be added to the
 214     filename and more than one file may be created.  The optional flag
 215     parameter has the same interpretation as the flag parameter of
 216     anydbm.open(). The optional protocol parameter specifies the
 217     version of the pickle protocol (0, 1, or 2).
 218
 219     See the module's __doc__ string for an overview of the interface.
 220     """
 221
 222     return DbfilenameShelf(filename, flag, protocol, writeback)