mirror/database.py

   1 # Copyright 2009 Google Inc. Released under the GPL v2
   2
   3 # This file contains the classes used for the known kernel versions persistent
   4 # storage
   5
   6 import cPickle, fcntl, os, tempfile
   7
   8
   9 class item(object):
  10     """Wrap a file item stored in a database."""
  11     def __init__(self, name, size, timestamp):
  12         assert type(size) == int
  13         assert type(timestamp) == int
  14
  15         self.name = name
  16         self.size = size
  17         self.timestamp = timestamp
  18
  19
  20     def __repr__(self):
  21         return ("database.item('%s', %d, %d)" %
  22                 (self.name, self.size, self.timestamp))
  23
  24
  25     def __eq__(self, other):
  26         if not isinstance(other, item):
  27             return NotImplemented
  28
  29         return (self.name == other.name and self.size == other.size and
  30                 self.timestamp == other.timestamp)
  31
  32
  33     def __ne__(self, other):
  34         return not self.__eq__(other)
  35
  36
  37 class database(object):
  38     """
  39     This is an Abstract Base Class for the file items database, not strictly
  40     needed in Python because of the dynamic nature of the language but useful
  41     to document the expected common API of the implementations.
  42     """
  43
  44     def get_dictionary(self):
  45         """
  46         Should be implemented to open and read the persistent contents of
  47         the database and return it as a key->value dictionary.
  48         """
  49         raise NotImplemented('get_dictionary not implemented')
  50
  51
  52     def merge_dictionary(self, values):
  53         """
  54         Should be implemented to merge the "values" dictionary into the
  55         database persistent contents (ie to update existent entries and to add
  56         those that do not exist).
  57         """
  58         raise NotImplemented('merge_dictionary not implemented')
  59
  60
  61 class dict_database(database):
  62     """
  63     A simple key->value database that uses standard python pickle dump of
  64     a dictionary object for persistent storage.
  65     """
  66     def __init__(self, path):
  67         self.path = path
  68
  69
  70     def get_dictionary(self, _open_func=open):
  71         """
  72         Return the key/value pairs as a standard dictionary.
  73         """
  74         try:
  75             fd = _open_func(self.path, 'rb')
  76         except IOError:
  77             # no db file, considering as if empty dictionary
  78             res = {}
  79         else:
  80             try:
  81                 res = cPickle.load(fd)
  82             finally:
  83                 fd.close()
  84
  85         return res
  86
  87
  88     def _aquire_lock(self):
  89         fd = os.open(self.path + '.lock', os.O_RDONLY | os.O_CREAT)
  90         try:
  91             # this may block
  92             fcntl.flock(fd, fcntl.LOCK_EX)
  93         except Exception, err:
  94             os.close(fd)
  95             raise err
  96
  97         return fd
  98
  99
 100     def merge_dictionary(self, values):
 101         """
 102         Merge the contents of "values" with the current contents of the
 103         database.
 104         """
 105         if not values:
 106             return
 107
 108         # use file locking to make the read/write of the file atomic
 109         lock_fd = self._aquire_lock()
 110
 111         # make sure we release locks in case of exceptions (in case the
 112         # process dies the OS will release them for us)
 113         try:
 114             contents = self.get_dictionary()
 115             contents.update(values)
 116
 117             # use a tempfile/atomic-rename technique to not require
 118             # synchronization for get_dictionary() calls and also protect
 119             # against full disk file corruption situations
 120             fd, fname = tempfile.mkstemp(prefix=os.path.basename(self.path),
 121                                          dir=os.path.dirname(self.path))
 122             write_file = os.fdopen(fd, 'wb')
 123             try:
 124                 try:
 125                     cPickle.dump(contents, write_file,
 126                                  protocol=cPickle.HIGHEST_PROTOCOL)
 127                 finally:
 128                     write_file.close()
 129
 130                 # this is supposed to be atomic on POSIX systems
 131                 os.rename(fname, self.path)
 132             except Exception:
 133                 os.unlink(fname)
 134                 raise
 135         finally:
 136             # close() releases any locks on that fd
 137             os.close(lock_fd)