Lib/bsddb/__init__.py

   1 #----------------------------------------------------------------------
   2 #  Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
   3 #  and Andrew Kuchling. All rights reserved.
   4 #
   5 #  Redistribution and use in source and binary forms, with or without
   6 #  modification, are permitted provided that the following conditions are
   7 #  met:
   8 #
   9 #    o Redistributions of source code must retain the above copyright
  10 #      notice, this list of conditions, and the disclaimer that follows.
  11 #
  12 #    o Redistributions in binary form must reproduce the above copyright
  13 #      notice, this list of conditions, and the following disclaimer in
  14 #      the documentation and/or other materials provided with the
  15 #      distribution.
  16 #
  17 #    o Neither the name of Digital Creations nor the names of its
  18 #      contributors may be used to endorse or promote products derived
  19 #      from this software without specific prior written permission.
  20 #
  21 #  THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
  22 #  IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  23 #  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  24 #  PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL
  25 #  CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26 #  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27 #  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  28 #  OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  29 #  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  30 #  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  31 #  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  32 #  DAMAGE.
  33 #----------------------------------------------------------------------
  34
  35
  36 """Support for Berkeley DB 4.0 through 4.7 with a simple interface.
  37
  38 For the full featured object oriented interface use the bsddb.db module
  39 instead.  It mirrors the Oracle Berkeley DB C API.
  40 """
  41
  42 import sys
  43 absolute_import = (sys.version_info[0] >= 3)
  44
  45 if sys.py3kwarning:
  46     import warnings
  47     warnings.warnpy3k("in 3.x, bsddb has been removed; "
  48                       "please use the pybsddb project instead",
  49                       DeprecationWarning, 2)
  50
  51 try:
  52     if __name__ == 'bsddb3':
  53         # import _pybsddb binary as it should be the more recent version from
  54         # a standalone pybsddb addon package than the version included with
  55         # python as bsddb._bsddb.
  56         if absolute_import :
  57             # Because this syntaxis is not valid before Python 2.5
  58             exec("from . import _pybsddb")
  59         else :
  60             import _pybsddb
  61         _bsddb = _pybsddb
  62         from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap
  63     else:
  64         import _bsddb
  65         from bsddb.dbutils import DeadlockWrap as _DeadlockWrap
  66 except ImportError:
  67     # Remove ourselves from sys.modules
  68     import sys
  69     del sys.modules[__name__]
  70     raise
  71
  72 # bsddb3 calls it db, but provide _db for backwards compatibility
  73 db = _db = _bsddb
  74 __version__ = db.__version__
  75
  76 error = db.DBError  # So bsddb.error will mean something...
  77
  78 #----------------------------------------------------------------------
  79
  80 import sys, os
  81
  82 from weakref import ref
  83
  84 if sys.version_info[0:2] <= (2, 5) :
  85     import UserDict
  86     MutableMapping = UserDict.DictMixin
  87 else :
  88     import collections
  89     MutableMapping = collections.MutableMapping
  90
  91 class _iter_mixin(MutableMapping):
  92     def _make_iter_cursor(self):
  93         cur = _DeadlockWrap(self.db.cursor)
  94         key = id(cur)
  95         self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
  96         return cur
  97
  98     def _gen_cref_cleaner(self, key):
  99         # use generate the function for the weakref callback here
 100         # to ensure that we do not hold a strict reference to cur
 101         # in the callback.
 102         return lambda ref: self._cursor_refs.pop(key, None)
 103
 104     def __iter__(self):
 105         self._kill_iteration = False
 106         self._in_iter += 1
 107         try:
 108             try:
 109                 cur = self._make_iter_cursor()
 110
 111                 # FIXME-20031102-greg: race condition.  cursor could
 112                 # be closed by another thread before this call.
 113
 114                 # since we're only returning keys, we call the cursor
 115                 # methods with flags=0, dlen=0, dofs=0
 116                 key = _DeadlockWrap(cur.first, 0,0,0)[0]
 117                 yield key
 118
 119                 next = getattr(cur, "next")
 120                 while 1:
 121                     try:
 122                         key = _DeadlockWrap(next, 0,0,0)[0]
 123                         yield key
 124                     except _bsddb.DBCursorClosedError:
 125                         if self._kill_iteration:
 126                             raise RuntimeError('Database changed size '
 127                                                'during iteration.')
 128                         cur = self._make_iter_cursor()
 129                         # FIXME-20031101-greg: race condition.  cursor could
 130                         # be closed by another thread before this call.
 131                         _DeadlockWrap(cur.set, key,0,0,0)
 132                         next = getattr(cur, "next")
 133             except _bsddb.DBNotFoundError:
 134                 pass
 135             except _bsddb.DBCursorClosedError:
 136                 # the database was modified during iteration.  abort.
 137                 pass
 138 # When Python 2.3 not supported in bsddb3, we can change this to "finally"
 139         except :
 140             self._in_iter -= 1
 141             raise
 142
 143         self._in_iter -= 1
 144
 145     def iteritems(self):
 146         if not self.db:
 147             return
 148         self._kill_iteration = False
 149         self._in_iter += 1
 150         try:
 151             try:
 152                 cur = self._make_iter_cursor()
 153
 154                 # FIXME-20031102-greg: race condition.  cursor could
 155                 # be closed by another thread before this call.
 156
 157                 kv = _DeadlockWrap(cur.first)
 158                 key = kv[0]
 159                 yield kv
 160
 161                 next = getattr(cur, "next")
 162                 while 1:
 163                     try:
 164                         kv = _DeadlockWrap(next)
 165                         key = kv[0]
 166                         yield kv
 167                     except _bsddb.DBCursorClosedError:
 168                         if self._kill_iteration:
 169                             raise RuntimeError('Database changed size '
 170                                                'during iteration.')
 171                         cur = self._make_iter_cursor()
 172                         # FIXME-20031101-greg: race condition.  cursor could
 173                         # be closed by another thread before this call.
 174                         _DeadlockWrap(cur.set, key,0,0,0)
 175                         next = getattr(cur, "next")
 176             except _bsddb.DBNotFoundError:
 177                 pass
 178             except _bsddb.DBCursorClosedError:
 179                 # the database was modified during iteration.  abort.
 180                 pass
 181 # When Python 2.3 not supported in bsddb3, we can change this to "finally"
 182         except :
 183             self._in_iter -= 1
 184             raise
 185
 186         self._in_iter -= 1
 187
 188
 189 class _DBWithCursor(_iter_mixin):
 190     """
 191     A simple wrapper around DB that makes it look like the bsddbobject in
 192     the old module.  It uses a cursor as needed to provide DB traversal.
 193     """
 194     def __init__(self, db):
 195         self.db = db
 196         self.db.set_get_returns_none(0)
 197
 198         # FIXME-20031101-greg: I believe there is still the potential
 199         # for deadlocks in a multithreaded environment if someone
 200         # attempts to use the any of the cursor interfaces in one
 201         # thread while doing a put or delete in another thread.  The
 202         # reason is that _checkCursor and _closeCursors are not atomic
 203         # operations.  Doing our own locking around self.dbc,
 204         # self.saved_dbc_key and self._cursor_refs could prevent this.
 205         # TODO: A test case demonstrating the problem needs to be written.
 206
 207         # self.dbc is a DBCursor object used to implement the
 208         # first/next/previous/last/set_location methods.
 209         self.dbc = None
 210         self.saved_dbc_key = None
 211
 212         # a collection of all DBCursor objects currently allocated
 213         # by the _iter_mixin interface.
 214         self._cursor_refs = {}
 215         self._in_iter = 0
 216         self._kill_iteration = False
 217
 218     def __del__(self):
 219         self.close()
 220
 221     def _checkCursor(self):
 222         if self.dbc is None:
 223             self.dbc = _DeadlockWrap(self.db.cursor)
 224             if self.saved_dbc_key is not None:
 225                 _DeadlockWrap(self.dbc.set, self.saved_dbc_key)
 226                 self.saved_dbc_key = None
 227
 228     # This method is needed for all non-cursor DB calls to avoid
 229     # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK
 230     # and DB_THREAD to be thread safe) when intermixing database
 231     # operations that use the cursor internally with those that don't.
 232     def _closeCursors(self, save=1):
 233         if self.dbc:
 234             c = self.dbc
 235             self.dbc = None
 236             if save:
 237                 try:
 238                     self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]
 239                 except db.DBError:
 240                     pass
 241             _DeadlockWrap(c.close)
 242             del c
 243         for cref in self._cursor_refs.values():
 244             c = cref()
 245             if c is not None:
 246                 _DeadlockWrap(c.close)
 247
 248     def _checkOpen(self):
 249         if self.db is None:
 250             raise error, "BSDDB object has already been closed"
 251
 252     def isOpen(self):
 253         return self.db is not None
 254
 255     def __len__(self):
 256         self._checkOpen()
 257         return _DeadlockWrap(lambda: len(self.db))  # len(self.db)
 258
 259     if sys.version_info[0:2] >= (2, 6) :
 260         def __repr__(self) :
 261             if self.isOpen() :
 262                 return repr(dict(_DeadlockWrap(self.db.items)))
 263             return repr(dict())
 264
 265     def __getitem__(self, key):
 266         self._checkOpen()
 267         return _DeadlockWrap(lambda: self.db[key])  # self.db[key]
 268
 269     def __setitem__(self, key, value):
 270         self._checkOpen()
 271         self._closeCursors()
 272         if self._in_iter and key not in self:
 273             self._kill_iteration = True
 274         def wrapF():
 275             self.db[key] = value
 276         _DeadlockWrap(wrapF)  # self.db[key] = value
 277
 278     def __delitem__(self, key):
 279         self._checkOpen()
 280         self._closeCursors()
 281         if self._in_iter and key in self:
 282             self._kill_iteration = True
 283         def wrapF():
 284             del self.db[key]
 285         _DeadlockWrap(wrapF)  # del self.db[key]
 286
 287     def close(self):
 288         self._closeCursors(save=0)
 289         if self.dbc is not None:
 290             _DeadlockWrap(self.dbc.close)
 291         v = 0
 292         if self.db is not None:
 293             v = _DeadlockWrap(self.db.close)
 294         self.dbc = None
 295         self.db = None
 296         return v
 297
 298     def keys(self):
 299         self._checkOpen()
 300         return _DeadlockWrap(self.db.keys)
 301
 302     def has_key(self, key):
 303         self._checkOpen()
 304         return _DeadlockWrap(self.db.has_key, key)
 305
 306     def set_location(self, key):
 307         self._checkOpen()
 308         self._checkCursor()
 309         return _DeadlockWrap(self.dbc.set_range, key)
 310
 311     def next(self):  # Renamed by "2to3"
 312         self._checkOpen()
 313         self._checkCursor()
 314         rv = _DeadlockWrap(getattr(self.dbc, "next"))
 315         return rv
 316
 317     if sys.version_info[0] >= 3 :  # For "2to3" conversion
 318         next = __next__
 319
 320     def previous(self):
 321         self._checkOpen()
 322         self._checkCursor()
 323         rv = _DeadlockWrap(self.dbc.prev)
 324         return rv
 325
 326     def first(self):
 327         self._checkOpen()
 328         # fix 1725856: don't needlessly try to restore our cursor position
 329         self.saved_dbc_key = None
 330         self._checkCursor()
 331         rv = _DeadlockWrap(self.dbc.first)
 332         return rv
 333
 334     def last(self):
 335         self._checkOpen()
 336         # fix 1725856: don't needlessly try to restore our cursor position
 337         self.saved_dbc_key = None
 338         self._checkCursor()
 339         rv = _DeadlockWrap(self.dbc.last)
 340         return rv
 341
 342     def sync(self):
 343         self._checkOpen()
 344         return _DeadlockWrap(self.db.sync)
 345
 346
 347 #----------------------------------------------------------------------
 348 # Compatibility object factory functions
 349
 350 def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,
 351             cachesize=None, lorder=None, hflags=0):
 352
 353     flags = _checkflag(flag, file)
 354     e = _openDBEnv(cachesize)
 355     d = db.DB(e)
 356     d.set_flags(hflags)
 357     if pgsize is not None:    d.set_pagesize(pgsize)
 358     if lorder is not None:    d.set_lorder(lorder)
 359     if ffactor is not None:   d.set_h_ffactor(ffactor)
 360     if nelem is not None:     d.set_h_nelem(nelem)
 361     d.open(file, db.DB_HASH, flags, mode)
 362     return _DBWithCursor(d)
 363
 364 #----------------------------------------------------------------------
 365
 366 def btopen(file, flag='c', mode=0666,
 367             btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,
 368             pgsize=None, lorder=None):
 369
 370     flags = _checkflag(flag, file)
 371     e = _openDBEnv(cachesize)
 372     d = db.DB(e)
 373     if pgsize is not None: d.set_pagesize(pgsize)
 374     if lorder is not None: d.set_lorder(lorder)
 375     d.set_flags(btflags)
 376     if minkeypage is not None: d.set_bt_minkey(minkeypage)
 377     if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)
 378     d.open(file, db.DB_BTREE, flags, mode)
 379     return _DBWithCursor(d)
 380
 381 #----------------------------------------------------------------------
 382
 383
 384 def rnopen(file, flag='c', mode=0666,
 385             rnflags=0, cachesize=None, pgsize=None, lorder=None,
 386             rlen=None, delim=None, source=None, pad=None):
 387
 388     flags = _checkflag(flag, file)
 389     e = _openDBEnv(cachesize)
 390     d = db.DB(e)
 391     if pgsize is not None: d.set_pagesize(pgsize)
 392     if lorder is not None: d.set_lorder(lorder)
 393     d.set_flags(rnflags)
 394     if delim is not None: d.set_re_delim(delim)
 395     if rlen is not None: d.set_re_len(rlen)
 396     if source is not None: d.set_re_source(source)
 397     if pad is not None: d.set_re_pad(pad)
 398     d.open(file, db.DB_RECNO, flags, mode)
 399     return _DBWithCursor(d)
 400
 401 #----------------------------------------------------------------------
 402
 403 def _openDBEnv(cachesize):
 404     e = db.DBEnv()
 405     if cachesize is not None:
 406         if cachesize >= 20480:
 407             e.set_cachesize(0, cachesize)
 408         else:
 409             raise error, "cachesize must be >= 20480"
 410     e.set_lk_detect(db.DB_LOCK_DEFAULT)
 411     e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)
 412     return e
 413
 414 def _checkflag(flag, file):
 415     if flag == 'r':
 416         flags = db.DB_RDONLY
 417     elif flag == 'rw':
 418         flags = 0
 419     elif flag == 'w':
 420         flags =  db.DB_CREATE
 421     elif flag == 'c':
 422         flags =  db.DB_CREATE
 423     elif flag == 'n':
 424         flags = db.DB_CREATE
 425         #flags = db.DB_CREATE | db.DB_TRUNCATE
 426         # we used db.DB_TRUNCATE flag for this before but Berkeley DB
 427         # 4.2.52 changed to disallowed truncate with txn environments.
 428         if file is not None and os.path.isfile(file):
 429             os.unlink(file)
 430     else:
 431         raise error, "flags should be one of 'r', 'w', 'c' or 'n'"
 432     return flags | db.DB_THREAD
 433
 434 #----------------------------------------------------------------------
 435
 436
 437 # This is a silly little hack that allows apps to continue to use the
 438 # DB_THREAD flag even on systems without threads without freaking out
 439 # Berkeley DB.
 440 #
 441 # This assumes that if Python was built with thread support then
 442 # Berkeley DB was too.
 443
 444 try:
 445     import thread
 446     del thread
 447 except ImportError:
 448     db.DB_THREAD = 0
 449
 450 #----------------------------------------------------------------------