zeroinstall/zerostore/__init__.py

   1 """
   2 Code for managing the implementation cache.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _, logger
   9 import os
  10
  11 from zeroinstall.support import basedir
  12 from zeroinstall import SafeException, support
  13
  14 class BadDigest(SafeException):
  15         """Thrown if a digest is invalid (either syntactically or cryptographically)."""
  16         detail = None
  17
  18 class NotStored(SafeException):
  19         """Throws if a requested implementation isn't in the cache."""
  20
  21 class NonwritableStore(SafeException):
  22         """Attempt to add to a non-writable store directory."""
  23
  24 def _copytree2(src, dst):
  25         import shutil
  26         names = os.listdir(src)
  27         assert os.path.isdir(dst)
  28         for name in names:
  29                 srcname = os.path.join(src, name)
  30                 dstname = os.path.join(dst, name)
  31                 if os.path.islink(srcname):
  32                         linkto = os.readlink(srcname)
  33                         os.symlink(linkto, dstname)
  34                 elif os.path.isdir(srcname):
  35                         os.mkdir(dstname)
  36                         mtime = int(os.lstat(srcname).st_mtime)
  37                         _copytree2(srcname, dstname)
  38                         os.utime(dstname, (mtime, mtime))
  39                 else:
  40                         shutil.copy2(srcname, dstname)
  41
  42 def _validate_pair(value):
  43         if '/' in value or \
  44            '\\' in value or \
  45            value.startswith('.'):
  46                 raise BadDigest("Invalid digest '{value}'".format(value = value))
  47
  48 def parse_algorithm_digest_pair(src):
  49         """Break apart an algorithm/digest into in a tuple.
  50         Old algorithms use '=' as the separator, while newer ones use '_'.
  51         @param src: the combined string
  52         @type src: str
  53         @return: the parsed values
  54         @rtype: (str, str)
  55         @raise BadDigest: if it can't be parsed
  56         @since: 1.10"""
  57         _validate_pair(src)
  58         if src.startswith('sha1=') or src.startswith('sha1new=') or src.startswith('sha256='):
  59                 return src.split('=', 1)
  60         result = src.split('_', 1)
  61         if len(result) != 2:
  62                 if '=' in src:
  63                         raise BadDigest("Use '_' not '=' for new algorithms, in {src}".format(src = src))
  64                 raise BadDigest("Can't parse digest {src}".format(src = src))
  65         return result
  66
  67 def format_algorithm_digest_pair(alg, digest):
  68         """The opposite of L{parse_algorithm_digest_pair}.
  69         The result is suitable for use as a directory name (does not contain '/' characters).
  70         @raise BadDigest: if the result is invalid
  71         @type alg: str
  72         @type digest: str
  73         @since: 1.10"""
  74         if alg in ('sha1', 'sha1new', 'sha256'):
  75                 result = alg + '=' + digest
  76         else:
  77                 result = alg + '_' + digest
  78         _validate_pair(result)
  79         return result
  80
  81 class Store:
  82         """A directory for storing implementations."""
  83
  84         def __init__(self, dir, public = False):
  85                 """Create a new Store.
  86                 @param dir: directory to contain the implementations
  87                 @type dir: str
  88                 @param public: deprecated
  89                 @type public: bool"""
  90                 self.dir = dir
  91
  92         def __str__(self):
  93                 return _("Store '%s'") % self.dir
  94
  95         def lookup(self, digest):
  96                 alg, value = parse_algorithm_digest_pair(digest)
  97                 dir = os.path.join(self.dir, digest)
  98                 if os.path.isdir(dir):
  99                         return dir
 100                 return None
 101
 102         def get_tmp_dir_for(self, required_digest):
 103                 """Create a temporary directory in the directory where we would store an implementation
 104                 with the given digest. This is used to setup a new implementation before being renamed if
 105                 it turns out OK.
 106                 @raise NonwritableStore: if we can't create it"""
 107                 try:
 108                         if not os.path.isdir(self.dir):
 109                                 os.makedirs(self.dir)
 110                         from tempfile import mkdtemp
 111                         tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
 112                         os.chmod(tmp, 0o755)    # r-x for all; needed by 0store-helper
 113                         return tmp
 114                 except OSError as ex:
 115                         raise NonwritableStore(str(ex))
 116
 117         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0, try_helper = False):
 118                 from . import unpack
 119
 120                 if self.lookup(required_digest):
 121                         logger.info(_("Not adding %s as it already exists!"), required_digest)
 122                         return
 123
 124                 tmp = self.get_tmp_dir_for(required_digest)
 125                 try:
 126                         unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset)
 127                 except:
 128                         import shutil
 129                         shutil.rmtree(tmp)
 130                         raise
 131
 132                 try:
 133                         self.check_manifest_and_rename(required_digest, tmp, extract, try_helper = try_helper)
 134                 except Exception:
 135                         #warn(_("Leaving extracted directory as %s"), tmp)
 136                         support.ro_rmtree(tmp)
 137                         raise
 138
 139         def add_dir_to_cache(self, required_digest, path, try_helper = False):
 140                 """Copy the contents of path to the cache.
 141                 @param required_digest: the expected digest
 142                 @type required_digest: str
 143                 @param path: the root of the tree to copy
 144                 @type path: str
 145                 @param try_helper: attempt to use privileged helper before user cache (since 0.26)
 146                 @type try_helper: bool
 147                 @raise BadDigest: if the contents don't match the given digest."""
 148                 if self.lookup(required_digest):
 149                         logger.info(_("Not adding %s as it already exists!"), required_digest)
 150                         return
 151
 152                 tmp = self.get_tmp_dir_for(required_digest)
 153                 try:
 154                         _copytree2(path, tmp)
 155                         self.check_manifest_and_rename(required_digest, tmp, try_helper = try_helper)
 156                 except:
 157                         logger.warn(_("Error importing directory."))
 158                         logger.warn(_("Deleting %s"), tmp)
 159                         support.ro_rmtree(tmp)
 160                         raise
 161
 162         def _add_with_helper(self, required_digest, path):
 163                 """Use 0store-secure-add to copy 'path' to the system store.
 164                 @param required_digest: the digest for path
 165                 @type required_digest: str
 166                 @param path: root of implementation directory structure
 167                 @type path: str
 168                 @return: True iff the directory was copied into the system cache successfully
 169                 """
 170                 if required_digest.startswith('sha1='):
 171                         return False            # Old digest alg not supported
 172                 helper = support.find_in_path('0store-secure-add-helper')
 173                 if not helper:
 174                         logger.info(_("'0store-secure-add-helper' command not found. Not adding to system cache."))
 175                         return False
 176                 import subprocess
 177                 env = os.environ.copy()
 178                 env['ENV_NOT_CLEARED'] = 'Unclean'      # (warn about insecure configurations)
 179                 env['HOME'] = 'Unclean'                 # (warn about insecure configurations)
 180                 dev_null = os.open(os.devnull, os.O_RDONLY)
 181                 try:
 182                         logger.info(_("Trying to add to system cache using %s"), helper)
 183                         child = subprocess.Popen([helper, required_digest],
 184                                                  stdin = dev_null,
 185                                                  cwd = path,
 186                                                  env = env)
 187                         exit_code = child.wait()
 188                 finally:
 189                         os.close(dev_null)
 190
 191                 if exit_code:
 192                         logger.warn(_("0store-secure-add-helper failed."))
 193                         return False
 194
 195                 logger.info(_("Added succcessfully."))
 196                 return True
 197
 198         def check_manifest_and_rename(self, required_digest, tmp, extract = None, try_helper = False):
 199                 """Check that tmp[/extract] has the required_digest.
 200                 On success, rename the checked directory to the digest, and
 201                 make the whole tree read-only.
 202                 @param try_helper: attempt to use privileged helper to import to system cache first (since 0.26)
 203                 @type try_helper: bool
 204                 @raise BadDigest: if the input directory doesn't match the given digest"""
 205                 if extract:
 206                         extracted = os.path.join(tmp, extract)
 207                         if not os.path.isdir(extracted):
 208                                 raise Exception(_('Directory %s not found in archive') % extract)
 209                 else:
 210                         extracted = tmp
 211
 212                 from . import manifest
 213
 214                 manifest.fixup_permissions(extracted)
 215
 216                 alg, required_value = manifest.splitID(required_digest)
 217                 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg))
 218                 if actual_digest != required_digest:
 219                         raise BadDigest(_('Incorrect manifest -- archive is corrupted.\n'
 220                                         'Required digest: %(required_digest)s\n'
 221                                         'Actual digest: %(actual_digest)s\n') %
 222                                         {'required_digest': required_digest, 'actual_digest': actual_digest})
 223
 224                 if try_helper:
 225                         if self._add_with_helper(required_digest, extracted):
 226                                 support.ro_rmtree(tmp)
 227                                 return
 228                         logger.info(_("Can't add to system store. Trying user store instead."))
 229
 230                 logger.info(_("Caching new implementation (digest %s) in %s"), required_digest, self.dir)
 231
 232                 final_name = os.path.join(self.dir, required_digest)
 233                 if os.path.isdir(final_name):
 234                         raise Exception(_("Item %s already stored.") % final_name) # XXX: not really an error
 235
 236                 # If we just want a subdirectory then the rename will change
 237                 # extracted/.. and so we'll need write permission on 'extracted'
 238
 239                 os.chmod(extracted, 0o755)
 240                 os.rename(extracted, final_name)
 241                 os.chmod(final_name, 0o555)
 242
 243                 if extract:
 244                         os.rmdir(tmp)
 245
 246         def __repr__(self):
 247                 return "<store: %s>" % self.dir
 248
 249 class Stores(object):
 250         """A list of L{Store}s. All stores are searched when looking for an implementation.
 251         When storing, we use the first of the system caches (if writable), or the user's
 252         cache otherwise."""
 253         __slots__ = ['stores']
 254
 255         def __init__(self):
 256                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 257                 self.stores = [Store(user_store)]
 258
 259                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 260                                                           'implementation-dirs')
 261                 logger.debug(_("Location of 'implementation-dirs' config file being used: '%s'"), impl_dirs)
 262                 if impl_dirs:
 263                         with open(impl_dirs, 'rt') as stream:
 264                                 dirs = stream.readlines()
 265                 else:
 266                         if os.name == "nt":
 267                                 from win32com.shell import shell, shellcon
 268                                 localAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0)
 269                                 commonAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_COMMON_APPDATA, 0, 0)
 270
 271                                 userCache = os.path.join(localAppData, "0install.net", "implementations")
 272                                 sharedCache = os.path.join(commonAppData, "0install.net", "implementations")
 273                                 dirs = [userCache, sharedCache]
 274
 275                         else:
 276                                 dirs = ['/var/cache/0install.net/implementations']
 277
 278                 for directory in dirs:
 279                         directory = directory.strip()
 280                         if directory and not directory.startswith('#'):
 281                                 logger.debug(_("Added system store '%s'"), directory)
 282                                 self.stores.append(Store(directory))
 283
 284         def lookup(self, digest):
 285                 """@deprecated: use lookup_any instead"""
 286                 return self.lookup_any([digest])
 287
 288         def lookup_any(self, digests):
 289                 """Search for digest in all stores.
 290                 @raises NotStored: if not found"""
 291                 path = self.lookup_maybe(digests)
 292                 if path:
 293                         return path
 294                 raise NotStored(_("Item with digests '%(digests)s' not found in stores. Searched:\n- %(stores)s") %
 295                         {'digests': digests, 'stores': '\n- '.join([s.dir for s in self.stores])})
 296
 297         def lookup_maybe(self, digests):
 298                 """Like lookup_any, but return None if it isn't found.
 299                 @since: 0.53"""
 300                 assert digests
 301                 for digest in digests:
 302                         assert digest
 303                         _validate_pair(digest)
 304                         for store in self.stores:
 305                                 path = store.lookup(digest)
 306                                 if path:
 307                                         return path
 308                 return None
 309
 310         def add_dir_to_cache(self, required_digest, dir):
 311                 """Add to the best writable cache.
 312                 @see: L{Store.add_dir_to_cache}"""
 313                 self._write_store(lambda store, **kwargs: store.add_dir_to_cache(required_digest, dir, **kwargs))
 314
 315         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
 316                 """Add to the best writable cache.
 317                 @see: L{Store.add_archive_to_cache}"""
 318                 self._write_store(lambda store, **kwargs: store.add_archive_to_cache(required_digest,
 319                                                 data, url, extract, type = type, start_offset = start_offset, **kwargs))
 320
 321         def _write_store(self, fn):
 322                 """Call fn(first_system_store). If it's read-only, try again with the user store."""
 323                 if len(self.stores) > 1:
 324                         try:
 325                                 fn(self.get_first_system_store())
 326                                 return
 327                         except NonwritableStore:
 328                                 logger.debug(_("%s not-writable. Trying helper instead."), self.get_first_system_store())
 329                                 pass
 330                 fn(self.stores[0], try_helper = True)
 331
 332         def get_first_system_store(self):
 333                 """The first system store is the one we try writing to first.
 334                 @since: 0.30"""
 335                 try:
 336                         return self.stores[1]
 337                 except IndexError:
 338                         raise SafeException(_("No system stores have been configured"))