zeroinstall/zerostore/__init__.py

   1 """
   2 Code for managing the implementation cache.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import debug, info, warn
  11
  12 from zeroinstall.support import basedir
  13 from zeroinstall import SafeException, support
  14
  15 class BadDigest(SafeException):
  16         """Thrown if a digest is invalid (either syntactically or cryptographically)."""
  17         detail = None
  18
  19 class NotStored(SafeException):
  20         """Throws if a requested implementation isn't in the cache."""
  21
  22 class NonwritableStore(SafeException):
  23         """Attempt to add to a non-writable store directory."""
  24
  25 def _copytree2(src, dst):
  26         import shutil
  27         names = os.listdir(src)
  28         assert os.path.isdir(dst)
  29         for name in names:
  30                 srcname = os.path.join(src, name)
  31                 dstname = os.path.join(dst, name)
  32                 if os.path.islink(srcname):
  33                         linkto = os.readlink(srcname)
  34                         os.symlink(linkto, dstname)
  35                 elif os.path.isdir(srcname):
  36                         os.mkdir(dstname)
  37                         mtime = int(os.lstat(srcname).st_mtime)
  38                         _copytree2(srcname, dstname)
  39                         os.utime(dstname, (mtime, mtime))
  40                 else:
  41                         shutil.copy2(srcname, dstname)
  42
  43 class Store:
  44         """A directory for storing implementations."""
  45
  46         def __init__(self, dir, public = False):
  47                 """Create a new Store.
  48                 @param dir: directory to contain the implementations
  49                 @type dir: str
  50                 @param public: deprecated
  51                 @type public: bool"""
  52                 self.dir = dir
  53
  54         def __str__(self):
  55                 return _("Store '%s'") % self.dir
  56
  57         def lookup(self, digest):
  58                 try:
  59                         alg, value = digest.split('=', 1)
  60                 except ValueError:
  61                         raise BadDigest(_("Digest must be in the form ALG=VALUE, not '%s'") % digest)
  62                 try:
  63                         assert '/' not in value
  64                         assert value not in ('', '.', '..')
  65                 except ValueError as ex:
  66                         raise BadDigest(_("Bad value for digest: %s") % str(ex))
  67                 dir = os.path.join(self.dir, digest)
  68                 if os.path.isdir(dir):
  69                         return dir
  70                 return None
  71
  72         def get_tmp_dir_for(self, required_digest):
  73                 """Create a temporary directory in the directory where we would store an implementation
  74                 with the given digest. This is used to setup a new implementation before being renamed if
  75                 it turns out OK.
  76                 @raise NonwritableStore: if we can't create it"""
  77                 try:
  78                         if not os.path.isdir(self.dir):
  79                                 os.makedirs(self.dir)
  80                         from tempfile import mkdtemp
  81                         tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
  82                         os.chmod(tmp, 0o755)    # r-x for all; needed by 0store-helper
  83                         return tmp
  84                 except OSError as ex:
  85                         raise NonwritableStore(str(ex))
  86
  87         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0, try_helper = False):
  88                 from . import unpack
  89
  90                 if self.lookup(required_digest):
  91                         info(_("Not adding %s as it already exists!"), required_digest)
  92                         return
  93
  94                 tmp = self.get_tmp_dir_for(required_digest)
  95                 try:
  96                         unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset)
  97                 except:
  98                         import shutil
  99                         shutil.rmtree(tmp)
 100                         raise
 101
 102                 try:
 103                         self.check_manifest_and_rename(required_digest, tmp, extract, try_helper = try_helper)
 104                 except Exception:
 105                         #warn(_("Leaving extracted directory as %s"), tmp)
 106                         support.ro_rmtree(tmp)
 107                         raise
 108
 109         def add_dir_to_cache(self, required_digest, path, try_helper = False):
 110                 """Copy the contents of path to the cache.
 111                 @param required_digest: the expected digest
 112                 @type required_digest: str
 113                 @param path: the root of the tree to copy
 114                 @type path: str
 115                 @param try_helper: attempt to use privileged helper before user cache (since 0.26)
 116                 @type try_helper: bool
 117                 @raise BadDigest: if the contents don't match the given digest."""
 118                 if self.lookup(required_digest):
 119                         info(_("Not adding %s as it already exists!"), required_digest)
 120                         return
 121
 122                 tmp = self.get_tmp_dir_for(required_digest)
 123                 try:
 124                         _copytree2(path, tmp)
 125                         self.check_manifest_and_rename(required_digest, tmp, try_helper = try_helper)
 126                 except:
 127                         warn(_("Error importing directory."))
 128                         warn(_("Deleting %s"), tmp)
 129                         support.ro_rmtree(tmp)
 130                         raise
 131
 132         def _add_with_helper(self, required_digest, path):
 133                 """Use 0store-secure-add to copy 'path' to the system store.
 134                 @param required_digest: the digest for path
 135                 @type required_digest: str
 136                 @param path: root of implementation directory structure
 137                 @type path: str
 138                 @return: True iff the directory was copied into the system cache successfully
 139                 """
 140                 if required_digest.startswith('sha1='):
 141                         return False            # Old digest alg not supported
 142                 helper = support.find_in_path('0store-secure-add-helper')
 143                 if not helper:
 144                         info(_("'0store-secure-add-helper' command not found. Not adding to system cache."))
 145                         return False
 146                 import subprocess
 147                 env = os.environ.copy()
 148                 env['ENV_NOT_CLEARED'] = 'Unclean'      # (warn about insecure configurations)
 149                 env['HOME'] = 'Unclean'                 # (warn about insecure configurations)
 150                 dev_null = os.open('/dev/null', os.O_RDONLY)
 151                 try:
 152                         info(_("Trying to add to system cache using %s"), helper)
 153                         child = subprocess.Popen([helper, required_digest],
 154                                                  stdin = dev_null,
 155                                                  cwd = path,
 156                                                  env = env)
 157                         exit_code = child.wait()
 158                 finally:
 159                         os.close(dev_null)
 160
 161                 if exit_code:
 162                         warn(_("0store-secure-add-helper failed."))
 163                         return False
 164
 165                 info(_("Added succcessfully."))
 166                 return True
 167
 168         def check_manifest_and_rename(self, required_digest, tmp, extract = None, try_helper = False):
 169                 """Check that tmp[/extract] has the required_digest.
 170                 On success, rename the checked directory to the digest, and
 171                 make the whole tree read-only.
 172                 @param try_helper: attempt to use privileged helper to import to system cache first (since 0.26)
 173                 @type try_helper: bool
 174                 @raise BadDigest: if the input directory doesn't match the given digest"""
 175                 if extract:
 176                         extracted = os.path.join(tmp, extract)
 177                         if not os.path.isdir(extracted):
 178                                 raise Exception(_('Directory %s not found in archive') % extract)
 179                 else:
 180                         extracted = tmp
 181
 182                 from . import manifest
 183
 184                 manifest.fixup_permissions(extracted)
 185
 186                 alg, required_value = manifest.splitID(required_digest)
 187                 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg))
 188                 if actual_digest != required_digest:
 189                         raise BadDigest(_('Incorrect manifest -- archive is corrupted.\n'
 190                                         'Required digest: %(required_digest)s\n'
 191                                         'Actual digest: %(actual_digest)s\n') %
 192                                         {'required_digest': required_digest, 'actual_digest': actual_digest})
 193
 194                 if try_helper:
 195                         if self._add_with_helper(required_digest, extracted):
 196                                 support.ro_rmtree(tmp)
 197                                 return
 198                         info(_("Can't add to system store. Trying user store instead."))
 199
 200                 info(_("Caching new implementation (digest %s) in %s"), required_digest, self.dir)
 201
 202                 final_name = os.path.join(self.dir, required_digest)
 203                 if os.path.isdir(final_name):
 204                         raise Exception(_("Item %s already stored.") % final_name) # XXX: not really an error
 205
 206                 # If we just want a subdirectory then the rename will change
 207                 # extracted/.. and so we'll need write permission on 'extracted'
 208
 209                 os.chmod(extracted, 0o755)
 210                 os.rename(extracted, final_name)
 211                 os.chmod(final_name, 0o555)
 212
 213                 if extract:
 214                         os.rmdir(tmp)
 215
 216         def __repr__(self):
 217                 return "<store: %s>" % self.dir
 218
 219 class Stores(object):
 220         """A list of L{Store}s. All stores are searched when looking for an implementation.
 221         When storing, we use the first of the system caches (if writable), or the user's
 222         cache otherwise."""
 223         __slots__ = ['stores']
 224
 225         def __init__(self):
 226                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 227                 self.stores = [Store(user_store)]
 228
 229                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 230                                                           'implementation-dirs')
 231                 debug(_("Location of 'implementation-dirs' config file being used: '%s'"), impl_dirs)
 232                 if impl_dirs:
 233                         dirs = open(impl_dirs)
 234                 else:
 235                         if os.name == "nt":
 236                                 from win32com.shell import shell, shellcon
 237                                 localAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0)
 238                                 commonAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_COMMON_APPDATA, 0, 0)
 239
 240                                 userCache = os.path.join(localAppData, "0install.net", "implementations")
 241                                 sharedCache = os.path.join(commonAppData, "0install.net", "implementations")
 242                                 dirs = [userCache, sharedCache]
 243
 244                         else:
 245                                 dirs = ['/var/cache/0install.net/implementations']
 246
 247                 for directory in dirs:
 248                         directory = directory.strip()
 249                         if directory and not directory.startswith('#'):
 250                                 debug(_("Added system store '%s'"), directory)
 251                                 self.stores.append(Store(directory))
 252
 253         def lookup(self, digest):
 254                 """@deprecated: use lookup_any instead"""
 255                 return self.lookup_any([digest])
 256
 257         def lookup_any(self, digests):
 258                 """Search for digest in all stores.
 259                 @raises NotStored: if not found"""
 260                 path = self.lookup_maybe(digests)
 261                 if path:
 262                         return path
 263                 raise NotStored(_("Item with digests '%(digests)s' not found in stores. Searched:\n- %(stores)s") %
 264                         {'digests': digests, 'stores': '\n- '.join([s.dir for s in self.stores])})
 265
 266         def lookup_maybe(self, digests):
 267                 """Like lookup_any, but return None if it isn't found.
 268                 @since: 0.53"""
 269                 assert digests
 270                 for digest in digests:
 271                         assert digest
 272                         if '/' in digest or '=' not in digest:
 273                                 raise BadDigest(_('Syntax error in digest (use ALG=VALUE, not %s)') % digest)
 274                         for store in self.stores:
 275                                 path = store.lookup(digest)
 276                                 if path:
 277                                         return path
 278                 return None
 279
 280         def add_dir_to_cache(self, required_digest, dir):
 281                 """Add to the best writable cache.
 282                 @see: L{Store.add_dir_to_cache}"""
 283                 self._write_store(lambda store, **kwargs: store.add_dir_to_cache(required_digest, dir, **kwargs))
 284
 285         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
 286                 """Add to the best writable cache.
 287                 @see: L{Store.add_archive_to_cache}"""
 288                 self._write_store(lambda store, **kwargs: store.add_archive_to_cache(required_digest,
 289                                                 data, url, extract, type = type, start_offset = start_offset, **kwargs))
 290
 291         def _write_store(self, fn):
 292                 """Call fn(first_system_store). If it's read-only, try again with the user store."""
 293                 if len(self.stores) > 1:
 294                         try:
 295                                 fn(self.get_first_system_store())
 296                                 return
 297                         except NonwritableStore:
 298                                 debug(_("%s not-writable. Trying helper instead."), self.get_first_system_store())
 299                                 pass
 300                 fn(self.stores[0], try_helper = True)
 301
 302         def get_first_system_store(self):
 303                 """The first system store is the one we try writing to first.
 304                 @since: 0.30"""
 305                 try:
 306                         return self.stores[1]
 307                 except IndexError:
 308                         raise SafeException(_("No system stores have been configured"))