2 #------------------------------------------------------------------------
3 # Copyright (c) 1997-2001 by Total Control Software
5 #------------------------------------------------------------------------
7 # Module Name: dbShelve.py
9 # Description: A reimplementation of the standard shelve.py that
10 # forces the use of cPickle, and DB.
12 # Creation Date: 11/3/97 3:39:04PM
14 # License: This is free software. You may use this software for any
15 # purpose including modification/redistribution, so long as
16 # this header remains intact and that you do not claim any
17 # rights of ownership or authorship of this software. This
18 # software has been tested, but no warranty is expressed or
21 # 13-Dec-2000: Updated to be used with the new bsddb3 package.
22 # Added DBShelfCursor class.
24 #------------------------------------------------------------------------
26 """Manage shelves of pickled objects using bsddb database files for the
30 #------------------------------------------------------------------------
36 absolute_import
= (sys
.version_info
[0] >= 3)
38 # Because this syntaxis is not valid before Python 2.5
39 exec("from . import db")
43 #At version 2.3 cPickle switched to using protocol instead of bin
44 if sys
.version_info
[:3] >= (2, 3, 0):
45 HIGHEST_PROTOCOL
= cPickle
.HIGHEST_PROTOCOL
46 # In python 2.3.*, "cPickle.dumps" accepts no
47 # named parameters. "pickle.dumps" accepts them,
48 # so this seems a bug.
49 if sys
.version_info
[:3] < (2, 4, 0):
50 def _dumps(object, protocol
):
51 return cPickle
.dumps(object, protocol
)
53 def _dumps(object, protocol
):
54 return cPickle
.dumps(object, protocol
=protocol
)
57 HIGHEST_PROTOCOL
= None
58 def _dumps(object, protocol
):
59 return cPickle
.dumps(object, bin
=protocol
)
62 if sys
.version_info
[0:2] <= (2, 5) :
64 from UserDict
import DictMixin
66 # DictMixin is new in Python 2.3
68 MutableMapping
= DictMixin
71 MutableMapping
= collections
.MutableMapping
73 #------------------------------------------------------------------------
76 def open(filename
, flags
=db
.DB_CREATE
, mode
=0660, filetype
=db
.DB_HASH
,
77 dbenv
=None, dbname
=None):
79 A simple factory function for compatibility with the standard
80 shleve.py module. It can be used like this, where key is a string
81 and data is a pickleable object:
83 from bsddb import dbshelve
84 db = dbshelve.open(filename)
90 if type(flags
) == type(''):
101 flags
= db
.DB_TRUNCATE | db
.DB_CREATE
103 raise db
.DBError
, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"
106 d
.open(filename
, dbname
, filetype
, flags
, mode
)
109 #---------------------------------------------------------------------------
111 class DBShelveError(db
.DBError
): pass
114 class DBShelf(MutableMapping
):
115 """A shelf to hold pickled objects, built upon a bsddb DB object. It
116 automatically pickles/unpickles data objects going to/from the DB.
118 def __init__(self
, dbenv
=None):
119 self
.db
= db
.DB(dbenv
)
122 self
.protocol
= HIGHEST_PROTOCOL
131 def __getattr__(self
, name
):
132 """Many methods we can just pass through to the DB object.
135 return getattr(self
.db
, name
)
138 #-----------------------------------
139 # Dictionary access methods
145 def __getitem__(self
, key
):
147 return cPickle
.loads(data
)
150 def __setitem__(self
, key
, value
):
151 data
= _dumps(value
, self
.protocol
)
155 def __delitem__(self
, key
):
159 def keys(self
, txn
=None):
161 return self
.db
.keys(txn
)
163 return self
.db
.keys()
165 if sys
.version_info
[0:2] >= (2, 6) :
167 return self
.db
.__iter
__()
170 def open(self
, *args
, **kwargs
):
171 self
.db
.open(*args
, **kwargs
)
175 def close(self
, *args
, **kwargs
):
176 self
.db
.close(*args
, **kwargs
)
182 return '<DBShelf @ 0x%x - closed>' % (id(self
))
184 return repr(dict(self
.iteritems()))
187 def items(self
, txn
=None):
189 items
= self
.db
.items(txn
)
191 items
= self
.db
.items()
195 newitems
.append( (k
, cPickle
.loads(v
)) )
198 def values(self
, txn
=None):
200 values
= self
.db
.values(txn
)
202 values
= self
.db
.values()
204 return map(cPickle
.loads
, values
)
206 #-----------------------------------
209 def __append(self
, value
, txn
=None):
210 data
= _dumps(value
, self
.protocol
)
211 return self
.db
.append(data
, txn
)
213 def append(self
, value
, txn
=None):
214 if self
.get_type() == db
.DB_RECNO
:
215 return self
.__append
(value
, txn
=txn
)
216 raise DBShelveError
, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"
219 def associate(self
, secondaryDB
, callback
, flags
=0):
220 def _shelf_callback(priKey
, priData
, realCallback
=callback
):
221 # Safe in Python 2.x because expresion short circuit
222 if sys
.version_info
[0] < 3 or isinstance(priData
, bytes
) :
223 data
= cPickle
.loads(priData
)
225 data
= cPickle
.loads(bytes(priData
, "iso8859-1")) # 8 bits
226 return realCallback(priKey
, data
)
228 return self
.db
.associate(secondaryDB
, _shelf_callback
, flags
)
231 #def get(self, key, default=None, txn=None, flags=0):
232 def get(self
, *args
, **kw
):
233 # We do it with *args and **kw so if the default value wasn't
234 # given nothing is passed to the extension module. That way
235 # an exception can be raised if set_get_returns_none is turned
237 data
= self
.db
.get(*args
, **kw
)
239 return cPickle
.loads(data
)
240 except (EOFError, TypeError, cPickle
.UnpicklingError
):
241 return data
# we may be getting the default value, or None,
242 # so it doesn't need unpickled.
244 def get_both(self
, key
, value
, txn
=None, flags
=0):
245 data
= _dumps(value
, self
.protocol
)
246 data
= self
.db
.get(key
, data
, txn
, flags
)
247 return cPickle
.loads(data
)
250 def cursor(self
, txn
=None, flags
=0):
251 c
= DBShelfCursor(self
.db
.cursor(txn
, flags
))
252 c
.protocol
= self
.protocol
256 def put(self
, key
, value
, txn
=None, flags
=0):
257 data
= _dumps(value
, self
.protocol
)
258 return self
.db
.put(key
, data
, txn
, flags
)
261 def join(self
, cursorList
, flags
=0):
262 raise NotImplementedError
265 #----------------------------------------------
266 # Methods allowed to pass-through to self.db
268 # close, delete, fd, get_byteswapped, get_type, has_key,
269 # key_range, open, remove, rename, stat, sync,
270 # upgrade, verify, and all set_* methods.
273 #---------------------------------------------------------------------------
278 def __init__(self
, cursor
):
285 def __getattr__(self
, name
):
286 """Some methods we can just pass through to the cursor object. (See below)"""
287 return getattr(self
.dbc
, name
)
290 #----------------------------------------------
292 def dup(self
, flags
=0):
293 c
= DBShelfCursor(self
.dbc
.dup(flags
))
294 c
.protocol
= self
.protocol
298 def put(self
, key
, value
, flags
=0):
299 data
= _dumps(value
, self
.protocol
)
300 return self
.dbc
.put(key
, data
, flags
)
303 def get(self
, *args
):
304 count
= len(args
) # a method overloading hack
305 method
= getattr(self
, 'get_%d' % count
)
308 def get_1(self
, flags
):
309 rec
= self
.dbc
.get(flags
)
310 return self
._extract
(rec
)
312 def get_2(self
, key
, flags
):
313 rec
= self
.dbc
.get(key
, flags
)
314 return self
._extract
(rec
)
316 def get_3(self
, key
, value
, flags
):
317 data
= _dumps(value
, self
.protocol
)
318 rec
= self
.dbc
.get(key
, flags
)
319 return self
._extract
(rec
)
322 def current(self
, flags
=0): return self
.get_1(flags|db
.DB_CURRENT
)
323 def first(self
, flags
=0): return self
.get_1(flags|db
.DB_FIRST
)
324 def last(self
, flags
=0): return self
.get_1(flags|db
.DB_LAST
)
325 def next(self
, flags
=0): return self
.get_1(flags|db
.DB_NEXT
)
326 def prev(self
, flags
=0): return self
.get_1(flags|db
.DB_PREV
)
327 def consume(self
, flags
=0): return self
.get_1(flags|db
.DB_CONSUME
)
328 def next_dup(self
, flags
=0): return self
.get_1(flags|db
.DB_NEXT_DUP
)
329 def next_nodup(self
, flags
=0): return self
.get_1(flags|db
.DB_NEXT_NODUP
)
330 def prev_nodup(self
, flags
=0): return self
.get_1(flags|db
.DB_PREV_NODUP
)
333 def get_both(self
, key
, value
, flags
=0):
334 data
= _dumps(value
, self
.protocol
)
335 rec
= self
.dbc
.get_both(key
, flags
)
336 return self
._extract
(rec
)
339 def set(self
, key
, flags
=0):
340 rec
= self
.dbc
.set(key
, flags
)
341 return self
._extract
(rec
)
343 def set_range(self
, key
, flags
=0):
344 rec
= self
.dbc
.set_range(key
, flags
)
345 return self
._extract
(rec
)
347 def set_recno(self
, recno
, flags
=0):
348 rec
= self
.dbc
.set_recno(recno
, flags
)
349 return self
._extract
(rec
)
353 def _extract(self
, rec
):
358 # Safe in Python 2.x because expresion short circuit
359 if sys
.version_info
[0] < 3 or isinstance(data
, bytes
) :
360 return key
, cPickle
.loads(data
)
362 return key
, cPickle
.loads(bytes(data
, "iso8859-1")) # 8 bits
364 #----------------------------------------------
365 # Methods allowed to pass-through to self.dbc
367 # close, count, delete, get_recno, join_item
370 #---------------------------------------------------------------------------