zeroinstall/zerostore/__init__.py

   1 """
   2 Code for managing the implementation cache.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import debug, info, warn
  11
  12 from zeroinstall.support import basedir
  13 from zeroinstall import SafeException, support
  14
  15 class BadDigest(SafeException):
  16         """Thrown if a digest is invalid (either syntactically or cryptographically)."""
  17         detail = None
  18
  19 class NotStored(SafeException):
  20         """Throws if a requested implementation isn't in the cache."""
  21
  22 class NonwritableStore(SafeException):
  23         """Attempt to add to a non-writable store directory."""
  24
  25 def _copytree2(src, dst):
  26         import shutil
  27         names = os.listdir(src)
  28         assert os.path.isdir(dst)
  29         for name in names:
  30                 srcname = os.path.join(src, name)
  31                 dstname = os.path.join(dst, name)
  32                 if os.path.islink(srcname):
  33                         linkto = os.readlink(srcname)
  34                         os.symlink(linkto, dstname)
  35                 elif os.path.isdir(srcname):
  36                         os.mkdir(dstname)
  37                         mtime = int(os.lstat(srcname).st_mtime)
  38                         _copytree2(srcname, dstname)
  39                         os.utime(dstname, (mtime, mtime))
  40                 else:
  41                         shutil.copy2(srcname, dstname)
  42
  43 def _validate_pair(value):
  44         if '/' in value or \
  45            '\\' in value or \
  46            value.startswith('.'):
  47                 raise BadDigest("Invalid digest '{value}'".format(value = value))
  48
  49 def parse_algorithm_digest_pair(src):
  50         """Break apart an algorithm/digest into in a tuple.
  51         Old algorithms use '=' as the separator, while newer ones use '_'.
  52         @param src: the combined string
  53         @type src: str
  54         @return: the parsed values
  55         @rtype: (str, str)
  56         @raise BadDigest: if it can't be parsed
  57         @since: 1.10"""
  58         _validate_pair(src)
  59         if src.startswith('sha1=') or src.startswith('sha1new=') or src.startswith('sha256='):
  60                 return src.split('=', 1)
  61         result = src.split('_', 1)
  62         if len(result) != 2:
  63                 if '=' in src:
  64                         raise BadDigest("Use '_' not '=' for new algorithms, in {src}".format(src = src))
  65                 raise BadDigest("Can't parse digest {src}".format(src = src))
  66         return result
  67
  68 def format_algorithm_digest_pair(alg, digest):
  69         """The opposite of L{parse_algorithm_digest_pair}.
  70         The result is suitable for use as a directory name (does not contain '/' characters).
  71         @raise BadDigest: if the result is invalid
  72         @type alg: str
  73         @type digest: str
  74         @since: 1.10"""
  75         if alg in ('sha1', 'sha1new', 'sha256'):
  76                 result = alg + '=' + digest
  77         else:
  78                 result = alg + '_' + digest
  79         _validate_pair(result)
  80         return result
  81
  82 class Store:
  83         """A directory for storing implementations."""
  84
  85         def __init__(self, dir, public = False):
  86                 """Create a new Store.
  87                 @param dir: directory to contain the implementations
  88                 @type dir: str
  89                 @param public: deprecated
  90                 @type public: bool"""
  91                 self.dir = dir
  92
  93         def __str__(self):
  94                 return _("Store '%s'") % self.dir
  95
  96         def lookup(self, digest):
  97                 alg, value = parse_algorithm_digest_pair(digest)
  98                 dir = os.path.join(self.dir, digest)
  99                 if os.path.isdir(dir):
 100                         return dir
 101                 return None
 102
 103         def get_tmp_dir_for(self, required_digest):
 104                 """Create a temporary directory in the directory where we would store an implementation
 105                 with the given digest. This is used to setup a new implementation before being renamed if
 106                 it turns out OK.
 107                 @raise NonwritableStore: if we can't create it"""
 108                 try:
 109                         if not os.path.isdir(self.dir):
 110                                 os.makedirs(self.dir)
 111                         from tempfile import mkdtemp
 112                         tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
 113                         os.chmod(tmp, 0o755)    # r-x for all; needed by 0store-helper
 114                         return tmp
 115                 except OSError as ex:
 116                         raise NonwritableStore(str(ex))
 117
 118         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0, try_helper = False):
 119                 from . import unpack
 120
 121                 if self.lookup(required_digest):
 122                         info(_("Not adding %s as it already exists!"), required_digest)
 123                         return
 124
 125                 tmp = self.get_tmp_dir_for(required_digest)
 126                 try:
 127                         unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset)
 128                 except:
 129                         import shutil
 130                         shutil.rmtree(tmp)
 131                         raise
 132
 133                 try:
 134                         self.check_manifest_and_rename(required_digest, tmp, extract, try_helper = try_helper)
 135                 except Exception:
 136                         #warn(_("Leaving extracted directory as %s"), tmp)
 137                         support.ro_rmtree(tmp)
 138                         raise
 139
 140         def add_dir_to_cache(self, required_digest, path, try_helper = False):
 141                 """Copy the contents of path to the cache.
 142                 @param required_digest: the expected digest
 143                 @type required_digest: str
 144                 @param path: the root of the tree to copy
 145                 @type path: str
 146                 @param try_helper: attempt to use privileged helper before user cache (since 0.26)
 147                 @type try_helper: bool
 148                 @raise BadDigest: if the contents don't match the given digest."""
 149                 if self.lookup(required_digest):
 150                         info(_("Not adding %s as it already exists!"), required_digest)
 151                         return
 152
 153                 tmp = self.get_tmp_dir_for(required_digest)
 154                 try:
 155                         _copytree2(path, tmp)
 156                         self.check_manifest_and_rename(required_digest, tmp, try_helper = try_helper)
 157                 except:
 158                         warn(_("Error importing directory."))
 159                         warn(_("Deleting %s"), tmp)
 160                         support.ro_rmtree(tmp)
 161                         raise
 162
 163         def _add_with_helper(self, required_digest, path):
 164                 """Use 0store-secure-add to copy 'path' to the system store.
 165                 @param required_digest: the digest for path
 166                 @type required_digest: str
 167                 @param path: root of implementation directory structure
 168                 @type path: str
 169                 @return: True iff the directory was copied into the system cache successfully
 170                 """
 171                 if required_digest.startswith('sha1='):
 172                         return False            # Old digest alg not supported
 173                 helper = support.find_in_path('0store-secure-add-helper')
 174                 if not helper:
 175                         info(_("'0store-secure-add-helper' command not found. Not adding to system cache."))
 176                         return False
 177                 import subprocess
 178                 env = os.environ.copy()
 179                 env['ENV_NOT_CLEARED'] = 'Unclean'      # (warn about insecure configurations)
 180                 env['HOME'] = 'Unclean'                 # (warn about insecure configurations)
 181                 dev_null = os.open(os.devnull, os.O_RDONLY)
 182                 try:
 183                         info(_("Trying to add to system cache using %s"), helper)
 184                         child = subprocess.Popen([helper, required_digest],
 185                                                  stdin = dev_null,
 186                                                  cwd = path,
 187                                                  env = env)
 188                         exit_code = child.wait()
 189                 finally:
 190                         os.close(dev_null)
 191
 192                 if exit_code:
 193                         warn(_("0store-secure-add-helper failed."))
 194                         return False
 195
 196                 info(_("Added succcessfully."))
 197                 return True
 198
 199         def check_manifest_and_rename(self, required_digest, tmp, extract = None, try_helper = False):
 200                 """Check that tmp[/extract] has the required_digest.
 201                 On success, rename the checked directory to the digest, and
 202                 make the whole tree read-only.
 203                 @param try_helper: attempt to use privileged helper to import to system cache first (since 0.26)
 204                 @type try_helper: bool
 205                 @raise BadDigest: if the input directory doesn't match the given digest"""
 206                 if extract:
 207                         extracted = os.path.join(tmp, extract)
 208                         if not os.path.isdir(extracted):
 209                                 raise Exception(_('Directory %s not found in archive') % extract)
 210                 else:
 211                         extracted = tmp
 212
 213                 from . import manifest
 214
 215                 manifest.fixup_permissions(extracted)
 216
 217                 alg, required_value = manifest.splitID(required_digest)
 218                 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg))
 219                 if actual_digest != required_digest:
 220                         raise BadDigest(_('Incorrect manifest -- archive is corrupted.\n'
 221                                         'Required digest: %(required_digest)s\n'
 222                                         'Actual digest: %(actual_digest)s\n') %
 223                                         {'required_digest': required_digest, 'actual_digest': actual_digest})
 224
 225                 if try_helper:
 226                         if self._add_with_helper(required_digest, extracted):
 227                                 support.ro_rmtree(tmp)
 228                                 return
 229                         info(_("Can't add to system store. Trying user store instead."))
 230
 231                 info(_("Caching new implementation (digest %s) in %s"), required_digest, self.dir)
 232
 233                 final_name = os.path.join(self.dir, required_digest)
 234                 if os.path.isdir(final_name):
 235                         raise Exception(_("Item %s already stored.") % final_name) # XXX: not really an error
 236
 237                 # If we just want a subdirectory then the rename will change
 238                 # extracted/.. and so we'll need write permission on 'extracted'
 239
 240                 os.chmod(extracted, 0o755)
 241                 os.rename(extracted, final_name)
 242                 os.chmod(final_name, 0o555)
 243
 244                 if extract:
 245                         os.rmdir(tmp)
 246
 247         def __repr__(self):
 248                 return "<store: %s>" % self.dir
 249
 250 class Stores(object):
 251         """A list of L{Store}s. All stores are searched when looking for an implementation.
 252         When storing, we use the first of the system caches (if writable), or the user's
 253         cache otherwise."""
 254         __slots__ = ['stores']
 255
 256         def __init__(self):
 257                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 258                 self.stores = [Store(user_store)]
 259
 260                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 261                                                           'implementation-dirs')
 262                 debug(_("Location of 'implementation-dirs' config file being used: '%s'"), impl_dirs)
 263                 if impl_dirs:
 264                         with open(impl_dirs, 'rt') as stream:
 265                                 dirs = stream.readlines()
 266                 else:
 267                         if os.name == "nt":
 268                                 from win32com.shell import shell, shellcon
 269                                 localAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0)
 270                                 commonAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_COMMON_APPDATA, 0, 0)
 271
 272                                 userCache = os.path.join(localAppData, "0install.net", "implementations")
 273                                 sharedCache = os.path.join(commonAppData, "0install.net", "implementations")
 274                                 dirs = [userCache, sharedCache]
 275
 276                         else:
 277                                 dirs = ['/var/cache/0install.net/implementations']
 278
 279                 for directory in dirs:
 280                         directory = directory.strip()
 281                         if directory and not directory.startswith('#'):
 282                                 debug(_("Added system store '%s'"), directory)
 283                                 self.stores.append(Store(directory))
 284
 285         def lookup(self, digest):
 286                 """@deprecated: use lookup_any instead"""
 287                 return self.lookup_any([digest])
 288
 289         def lookup_any(self, digests):
 290                 """Search for digest in all stores.
 291                 @raises NotStored: if not found"""
 292                 path = self.lookup_maybe(digests)
 293                 if path:
 294                         return path
 295                 raise NotStored(_("Item with digests '%(digests)s' not found in stores. Searched:\n- %(stores)s") %
 296                         {'digests': digests, 'stores': '\n- '.join([s.dir for s in self.stores])})
 297
 298         def lookup_maybe(self, digests):
 299                 """Like lookup_any, but return None if it isn't found.
 300                 @since: 0.53"""
 301                 assert digests
 302                 for digest in digests:
 303                         assert digest
 304                         _validate_pair(digest)
 305                         for store in self.stores:
 306                                 path = store.lookup(digest)
 307                                 if path:
 308                                         return path
 309                 return None
 310
 311         def add_dir_to_cache(self, required_digest, dir):
 312                 """Add to the best writable cache.
 313                 @see: L{Store.add_dir_to_cache}"""
 314                 self._write_store(lambda store, **kwargs: store.add_dir_to_cache(required_digest, dir, **kwargs))
 315
 316         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
 317                 """Add to the best writable cache.
 318                 @see: L{Store.add_archive_to_cache}"""
 319                 self._write_store(lambda store, **kwargs: store.add_archive_to_cache(required_digest,
 320                                                 data, url, extract, type = type, start_offset = start_offset, **kwargs))
 321
 322         def _write_store(self, fn):
 323                 """Call fn(first_system_store). If it's read-only, try again with the user store."""
 324                 if len(self.stores) > 1:
 325                         try:
 326                                 fn(self.get_first_system_store())
 327                                 return
 328                         except NonwritableStore:
 329                                 debug(_("%s not-writable. Trying helper instead."), self.get_first_system_store())
 330                                 pass
 331                 fn(self.stores[0], try_helper = True)
 332
 333         def get_first_system_store(self):
 334                 """The first system store is the one we try writing to first.
 335                 @since: 0.30"""
 336                 try:
 337                         return self.stores[1]
 338                 except IndexError:
 339                         raise SafeException(_("No system stores have been configured"))