zeroinstall/zerostore/__init__.py

   1 """
   2 Code for managing the implementation cache.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 import os
   9 from logging import debug, info, warn
  10
  11 from zeroinstall.support import basedir
  12 from zeroinstall import SafeException, support
  13
  14 class BadDigest(SafeException):
  15         """Thrown if a digest is invalid (either syntactically or cryptographically)."""
  16         detail = None
  17
  18 class NotStored(SafeException):
  19         """Throws if a requested implementation isn't in the cache."""
  20
  21 class NonwritableStore(SafeException):
  22         """Attempt to add to a non-writable store directory."""
  23
  24 def _copytree2(src, dst):
  25         import shutil
  26         names = os.listdir(src)
  27         assert os.path.isdir(dst)
  28         errors = []
  29         for name in names:
  30                 srcname = os.path.join(src, name)
  31                 dstname = os.path.join(dst, name)
  32                 if os.path.islink(srcname):
  33                         linkto = os.readlink(srcname)
  34                         os.symlink(linkto, dstname)
  35                 elif os.path.isdir(srcname):
  36                         os.mkdir(dstname)
  37                         mtime = int(os.lstat(srcname).st_mtime)
  38                         _copytree2(srcname, dstname)
  39                         os.utime(dstname, (mtime, mtime))
  40                 else:
  41                         shutil.copy2(srcname, dstname)
  42
  43 class Store:
  44         """A directory for storing implementations."""
  45
  46         def __init__(self, dir, public = False):
  47                 """Create a new Store.
  48                 @param dir: directory to contain the implementations
  49                 @type dir: str
  50                 @param public: deprecated
  51                 @type public: bool"""
  52                 self.dir = dir
  53
  54         def __str__(self):
  55                 return "Store '%s'" % self.dir
  56
  57         def lookup(self, digest):
  58                 try:
  59                         alg, value = digest.split('=', 1)
  60                 except ValueError:
  61                         raise BadDigest("Digest must be in the form ALG=VALUE, not '%s'" % digest)
  62                 try:
  63                         assert '/' not in value
  64                         int(value, 16)          # Check valid format
  65                 except ValueError, ex:
  66                         raise BadDigest("Bad value for digest: %s" % str(ex))
  67                 dir = os.path.join(self.dir, digest)
  68                 if os.path.isdir(dir):
  69                         return dir
  70                 return None
  71
  72         def get_tmp_dir_for(self, required_digest):
  73                 """Create a temporary directory in the directory where we would store an implementation
  74                 with the given digest. This is used to setup a new implementation before being renamed if
  75                 it turns out OK.
  76                 @raise NonwritableStore: if we can't create it"""
  77                 try:
  78                         if not os.path.isdir(self.dir):
  79                                 os.makedirs(self.dir)
  80                         from tempfile import mkdtemp
  81                         tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
  82                         os.chmod(tmp, 0755)     # r-x for all; needed by 0store-helper
  83                         return tmp
  84                 except OSError, ex:
  85                         raise NonwritableStore(str(ex))
  86
  87         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0, try_helper = False):
  88                 import unpack
  89                 info("Caching new implementation (digest %s)", required_digest)
  90
  91                 if self.lookup(required_digest):
  92                         info("Not adding %s as it already exists!", required_digest)
  93                         return
  94
  95                 tmp = self.get_tmp_dir_for(required_digest)
  96                 try:
  97                         unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset)
  98                 except:
  99                         import shutil
 100                         shutil.rmtree(tmp)
 101                         raise
 102
 103                 try:
 104                         self.check_manifest_and_rename(required_digest, tmp, extract, try_helper = try_helper)
 105                 except Exception, ex:
 106                         warn("Leaving extracted directory as %s", tmp)
 107                         raise
 108
 109         def add_dir_to_cache(self, required_digest, path, try_helper = False):
 110                 """Copy the contents of path to the cache.
 111                 @param required_digest: the expected digest
 112                 @type required_digest: str
 113                 @param path: the root of the tree to copy
 114                 @type path: str
 115                 @param try_helper: attempt to use privileged helper before user cache (since 0.26)
 116                 @type try_helper: bool
 117                 @raise BadDigest: if the contents don't match the given digest."""
 118                 if self.lookup(required_digest):
 119                         info("Not adding %s as it already exists!", required_digest)
 120                         return
 121
 122                 tmp = self.get_tmp_dir_for(required_digest)
 123                 try:
 124                         _copytree2(path, tmp)
 125                         self.check_manifest_and_rename(required_digest, tmp, try_helper = try_helper)
 126                 except:
 127                         warn("Error importing directory.")
 128                         warn("Deleting %s", tmp)
 129                         support.ro_rmtree(tmp)
 130                         raise
 131
 132         def _add_with_helper(self, required_digest, path):
 133                 """Use 0store-secure-add to copy 'path' to the system store.
 134                 @param required_digest: the digest for path
 135                 @type required_digest: str
 136                 @param path: root of implementation directory structure
 137                 @type path: str
 138                 @return: True iff the directory was copied into the system cache successfully
 139                 """
 140                 if required_digest.startswith('sha1='):
 141                         return False            # Old digest alg not supported
 142                 helper = support.find_in_path('0store-secure-add-helper')
 143                 if not helper:
 144                         info("'0store-secure-add-helper' command not found. Not adding to system cache.")
 145                         return False
 146                 import subprocess
 147                 env = os.environ.copy()
 148                 env['ENV_NOT_CLEARED'] = 'Unclean'      # (warn about insecure configurations)
 149                 dev_null = os.open('/dev/null', os.O_RDONLY)
 150                 try:
 151                         info("Trying to add to system cache using %s", helper)
 152                         child = subprocess.Popen([helper, required_digest],
 153                                                  stdin = dev_null,
 154                                                  cwd = path,
 155                                                  env = env)
 156                         exit_code = child.wait()
 157                 finally:
 158                         os.close(dev_null)
 159
 160                 if exit_code:
 161                         warn("0store-secure-add-helper failed.")
 162                         return False
 163
 164                 info("Added succcessfully.")
 165                 return True
 166
 167         def check_manifest_and_rename(self, required_digest, tmp, extract = None, try_helper = False):
 168                 """Check that tmp[/extract] has the required_digest.
 169                 On success, rename the checked directory to the digest, and
 170                 make the whole tree read-only.
 171                 @param try_helper: attempt to use privileged helper to import to system cache first (since 0.26)
 172                 @type try_helper: bool
 173                 @raise BadDigest: if the input directory doesn't match the given digest"""
 174                 if extract:
 175                         extracted = os.path.join(tmp, extract)
 176                         if not os.path.isdir(extracted):
 177                                 raise Exception('Directory %s not found in archive' % extract)
 178                 else:
 179                         extracted = tmp
 180
 181                 import manifest
 182
 183                 manifest.fixup_permissions(extracted)
 184
 185                 alg, required_value = manifest.splitID(required_digest)
 186                 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg))
 187                 if actual_digest != required_digest:
 188                         raise BadDigest('Incorrect manifest -- archive is corrupted.\n'
 189                                         'Required digest: %s\n'
 190                                         'Actual digest: %s\n' %
 191                                         (required_digest, actual_digest))
 192
 193                 if try_helper:
 194                         if self._add_with_helper(required_digest, extracted):
 195                                 support.ro_rmtree(tmp)
 196                                 return
 197                         info("Can't add to system store. Trying user store instead.")
 198
 199                 final_name = os.path.join(self.dir, required_digest)
 200                 if os.path.isdir(final_name):
 201                         raise Exception("Item %s already stored." % final_name) # XXX: not really an error
 202
 203                 # If we just want a subdirectory then the rename will change
 204                 # extracted/.. and so we'll need write permission on 'extracted'
 205
 206                 os.chmod(extracted, 0755)
 207                 os.rename(extracted, final_name)
 208                 os.chmod(final_name, 0555)
 209
 210                 if extract:
 211                         os.rmdir(tmp)
 212
 213 class Stores(object):
 214         """A list of L{Store}s. All stores are searched when looking for an implementation.
 215         When storing, we use the first of the system caches (if writable), or the user's
 216         cache otherwise."""
 217         __slots__ = ['stores']
 218
 219         def __init__(self):
 220                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 221                 self.stores = [Store(user_store)]
 222
 223                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 224                                                           'implementation-dirs')
 225                 debug("Location of 'implementation-dirs' config file being used: '%s'", impl_dirs)
 226                 if impl_dirs:
 227                         dirs = file(impl_dirs)
 228                 else:
 229                         dirs = ['/var/cache/0install.net/implementations']
 230                 for directory in dirs:
 231                         directory = directory.strip()
 232                         if directory and not directory.startswith('#'):
 233                                 debug("Added system store '%s'", directory)
 234                                 self.stores.append(Store(directory))
 235
 236         def lookup(self, digest):
 237                 """Search for digest in all stores."""
 238                 assert digest
 239                 if '/' in digest or '=' not in digest:
 240                         raise BadDigest('Syntax error in digest (use ALG=VALUE, not %s)' % digest)
 241                 for store in self.stores:
 242                         path = store.lookup(digest)
 243                         if path:
 244                                 return path
 245                 raise NotStored("Item with digest '%s' not found in stores. Searched:\n- %s" %
 246                         (digest, '\n- '.join([s.dir for s in self.stores])))
 247
 248         def add_dir_to_cache(self, required_digest, dir):
 249                 """Add to the best writable cache.
 250                 @see: L{Store.add_dir_to_cache}"""
 251                 self._write_store(lambda store, **kwargs: store.add_dir_to_cache(required_digest, dir, **kwargs))
 252
 253         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
 254                 """Add to the best writable cache.
 255                 @see: L{Store.add_archive_to_cache}"""
 256                 self._write_store(lambda store, **kwargs: store.add_archive_to_cache(required_digest,
 257                                                 data, url, extract, type = type, start_offset = start_offset, **kwargs))
 258
 259         def _write_store(self, fn):
 260                 """Call fn(first_system_store). If it's read-only, try again with the user store."""
 261                 if len(self.stores) > 1:
 262                         try:
 263                                 fn(self.get_first_system_store())
 264                                 return
 265                         except NonwritableStore:
 266                                 debug("%s not-writable. Trying helper instead.", self.get_first_system_store())
 267                                 pass
 268                 fn(self.stores[0], try_helper = True)
 269
 270         def get_first_system_store(self):
 271                 """The first system store is the one we try writing to first.
 272                 @since: 0.30"""
 273                 try:
 274                         return self.stores[1]
 275                 except IndexError:
 276                         raise SafeException("No system stores have been configured")