zeroinstall/zerostore/__init__.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os
   5 from logging import debug, info, warn
   6
   7 from zeroinstall.injector import basedir
   8 from zeroinstall import SafeException
   9
  10 class BadDigest(SafeException):
  11         detail = None
  12 class NotStored(SafeException): pass
  13
  14 def copytree2(src, dst):
  15         import shutil
  16         names = os.listdir(src)
  17         assert os.path.isdir(dst)
  18         errors = []
  19         for name in names:
  20                 srcname = os.path.join(src, name)
  21                 dstname = os.path.join(dst, name)
  22                 if os.path.islink(srcname):
  23                         linkto = os.readlink(srcname)
  24                         os.symlink(linkto, dstname)
  25                 elif os.path.isdir(srcname):
  26                         os.mkdir(dstname)
  27                         mtime = os.lstat(srcname).st_mtime
  28                         copytree2(srcname, dstname)
  29                         os.utime(dstname, (mtime, mtime))
  30                 else:
  31                         shutil.copy2(srcname, dstname)
  32
  33 class Store:
  34         def __init__(self, dir):
  35                 self.dir = dir
  36
  37         def lookup(self, digest):
  38                 alg, value = digest.split('=', 1)
  39                 assert alg in ('sha1', 'sha1new', 'sha256')
  40                 assert '/' not in value
  41                 int(value, 16)          # Check valid format
  42                 dir = os.path.join(self.dir, digest)
  43                 if os.path.isdir(dir):
  44                         return dir
  45                 return None
  46
  47         def get_tmp_dir_for(self, required_digest):
  48                 """Create a temporary directory in the directory where we would store an implementation
  49                 with the given digest. This is used to setup a new implementation before being renamed if
  50                 it turns out OK."""
  51                 if not os.path.isdir(self.dir):
  52                         os.makedirs(self.dir)
  53                 from tempfile import mkdtemp
  54                 tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
  55                 return tmp
  56
  57         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
  58                 import unpack
  59                 info("Caching new implementation (digest %s)", required_digest)
  60
  61                 if self.lookup(required_digest):
  62                         info("Not adding %s as it already exists!", required_digest)
  63                         return
  64
  65                 tmp = self.get_tmp_dir_for(required_digest)
  66                 try:
  67                         unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset)
  68                 except:
  69                         import shutil
  70                         shutil.rmtree(tmp)
  71                         raise
  72
  73                 try:
  74                         self.check_manifest_and_rename(required_digest, tmp, extract)
  75                 except Exception, ex:
  76                         warn("Leaving extracted directory as %s", tmp)
  77                         raise
  78
  79         def add_dir_to_cache(self, required_digest, path):
  80                 if self.lookup(required_digest):
  81                         info("Not adding %s as it already exists!", required_digest)
  82                         return
  83
  84                 if not os.path.isdir(self.dir):
  85                         os.makedirs(self.dir)
  86                 from tempfile import mkdtemp
  87                 tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
  88                 copytree2(path, tmp)
  89                 try:
  90                         self.check_manifest_and_rename(required_digest, tmp)
  91                 except:
  92                         warn("Error importing directory.")
  93                         warn("Deleting %s", tmp)
  94                         import shutil
  95                         shutil.rmtree(tmp)
  96                         raise
  97
  98         def check_manifest_and_rename(self, required_digest, tmp, extract = None):
  99                 if extract:
 100                         extracted = os.path.join(tmp, extract)
 101                         if not os.path.isdir(extracted):
 102                                 raise Exception('Directory %s not found in archive' % extract)
 103                 else:
 104                         extracted = tmp
 105
 106                 import manifest
 107                 alg, required_value = manifest.splitID(required_digest)
 108                 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg))
 109                 if actual_digest != required_digest:
 110                         raise BadDigest('Incorrect manifest -- archive is corrupted.\n'
 111                                         'Required digest: %s\n'
 112                                         'Actual digest: %s\n' %
 113                                         (required_digest, actual_digest))
 114
 115                 final_name = os.path.join(self.dir, required_digest)
 116                 if os.path.isdir(final_name):
 117                         raise Exception("Item %s already stored." % final_name)
 118                 if extract:
 119                         os.rename(os.path.join(tmp, extract), final_name)
 120                         os.rmdir(tmp)
 121                 else:
 122                         os.rename(tmp, final_name)
 123
 124 class Stores(object):
 125         __slots__ = ['stores']
 126
 127         def __init__(self):
 128                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 129                 self.stores = [Store(user_store)]
 130
 131                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 132                                                           'implementation-dirs')
 133                 debug("Location of 'implementation-dirs' config file being used: '%s'", impl_dirs)
 134                 if impl_dirs:
 135                         dirs = file(impl_dirs)
 136                 else:
 137                         dirs = ['/var/cache/0install.net/implementations']
 138                 for directory in dirs:
 139                         directory = directory.strip()
 140                         if directory and not directory.startswith('#'):
 141                                 if os.path.isdir(directory):
 142                                         self.stores.append(Store(directory))
 143                                         debug("Added system store '%s'", directory)
 144                                 else:
 145                                         info("Ignoring non-directory store '%s'", directory)
 146
 147         def lookup(self, digest):
 148                 """Search for digest in all stores."""
 149                 assert digest
 150                 if '/' in digest or '=' not in digest:
 151                         raise BadDigest('Syntax error in digest (use ALG=VALUE)')
 152                 for store in self.stores:
 153                         path = store.lookup(digest)
 154                         if path:
 155                                 return path
 156                 raise NotStored("Item with digest '%s' not found in stores. Searched:\n- %s" %
 157                         (digest, '\n- '.join([s.dir for s in self.stores])))
 158
 159         def add_dir_to_cache(self, required_digest, dir):
 160                 self.stores[0].add_dir_to_cache(required_digest, dir)
 161
 162         def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):
 163                 self.stores[0].add_archive_to_cache(required_digest, data, url, extract, type = type, start_offset = start_offset)