zeroinstall/zerostore/__init__.py

   1 import os
   2 import shutil
   3 import traceback
   4 from tempfile import mkdtemp
   5 import sha
   6 import re
   7 from logging import debug, info, warn
   8
   9 import manifest
  10 from zeroinstall.injector import basedir
  11
  12 class BadDigest(Exception): pass
  13 class NotStored(Exception): pass
  14
  15 _recent_gnu_tar = None
  16 def recent_gnu_tar():
  17         global _recent_gnu_tar
  18         if _recent_gnu_tar is None:
  19                 _recent_gnu_tar = False
  20                 version = os.popen('tar --version 2>&1').next()
  21                 if '(GNU tar)' in version:
  22                         try:
  23                                 version = version.split(')', 1)[1].strip()
  24                                 assert version
  25                                 version = map(int, version.split('.'))
  26                                 _recent_gnu_tar = version > [1, 14, 0]
  27                         except:
  28                                 warn("Failed to extract GNU tar version number")
  29         return _recent_gnu_tar
  30
  31 def copytree2(src, dst):
  32         names = os.listdir(src)
  33         assert os.path.isdir(dst)
  34         errors = []
  35         for name in names:
  36                 srcname = os.path.join(src, name)
  37                 dstname = os.path.join(dst, name)
  38                 if os.path.islink(srcname):
  39                         linkto = os.readlink(srcname)
  40                         os.symlink(linkto, dstname)
  41                 elif os.path.isdir(srcname):
  42                         os.mkdir(dstname)
  43                         mtime = os.lstat(srcname).st_mtime
  44                         copytree2(srcname, dstname)
  45                         os.utime(dstname, (mtime, mtime))
  46                 else:
  47                         shutil.copy2(srcname, dstname)
  48
  49 class Store:
  50         def __init__(self, dir):
  51                 self.dir = dir
  52
  53         def lookup(self, digest):
  54                 alg, value = digest.split('=', 1)
  55                 assert alg == 'sha1'
  56                 assert '/' not in value
  57                 int(value, 16)          # Check valid format
  58                 dir = os.path.join(self.dir, digest)
  59                 if os.path.isdir(dir):
  60                         return dir
  61                 return None
  62
  63         def add_archive_to_cache(self, required_digest, data, url, extract = None):
  64                 if url.endswith('.tar.bz2'):
  65                         self.add_tbz_to_cache(required_digest, data, extract)
  66                 else:
  67                         if not (url.endswith('.tar.gz') or url.endswith('.tgz')):
  68                                 warn('Unknown extension on "%s"; assuming tar.gz format' % url)
  69                         self.add_tgz_to_cache(required_digest, data, extract)
  70
  71         def add_tbz_to_cache(self, required_digest, data, extract = None):
  72                 self.add_tar_to_cache(required_digest, data, extract, '--bzip2')
  73
  74         def add_tgz_to_cache(self, required_digest, data, extract = None):
  75                 self.add_tar_to_cache(required_digest, data, extract, '-z')
  76
  77         def add_tar_to_cache(self, required_digest, data, extract, decompress):
  78                 """Data is a .tgz compressed archive. Extract it somewhere, check that
  79                 the digest is correct, and add it to the store.
  80                 extract is the name of a directory within the archive to extract, rather
  81                 than extracting the whole archive. This is most useful to remove an extra
  82                 top-level directory."""
  83                 assert required_digest.startswith('sha1=')
  84                 info("Caching new implementation (digest %s)", required_digest)
  85
  86                 if self.lookup(required_digest):
  87                         info("Not adding %s as it already exists!", required_digest)
  88                         return
  89
  90                 if recent_gnu_tar():
  91                         args = ['tar', decompress, '-x', '--no-same-owner', '--no-same-permissions']
  92                 else:
  93                         args = ['tar', decompress, '-xf', '-']
  94
  95                 if extract:
  96                         # Limit the characters we accept, to avoid sending dodgy
  97                         # strings to tar
  98                         if not re.match('^[a-zA-Z0-9][-_a-zA-Z0-9.]*$', extract):
  99                                 raise Exception('Illegal character in extract attribute')
 100                         args.append(extract)
 101
 102                 tmp = self.extract(data, args)
 103                 try:
 104                         self.check_manifest_and_rename(required_digest, tmp, extract)
 105                 except Exception, ex:
 106                         warn("Leaving extracted directory as %s", tmp)
 107                         raise
 108
 109         def add_dir_to_cache(self, required_digest, path):
 110                 if self.lookup(required_digest):
 111                         info("Not adding %s as it already exists!", required_digest)
 112                         return
 113
 114                 if not os.path.isdir(self.dir):
 115                         os.makedirs(self.dir)
 116                 tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
 117                 copytree2(path, tmp)
 118                 try:
 119                         self.check_manifest_and_rename(required_digest, tmp)
 120                 except:
 121                         warn("Error importing directory.")
 122                         warn("Deleting %s", tmp)
 123                         shutil.rmtree(tmp)
 124                         raise
 125
 126         def check_manifest_and_rename(self, required_digest, tmp, extract = None):
 127                 if extract:
 128                         extracted = os.path.join(tmp, extract)
 129                         if not os.path.isdir(extracted):
 130                                 raise Exception('Directory %s not found in archive' % extract)
 131                 else:
 132                         extracted = tmp
 133
 134                 sha1 = 'sha1=' + manifest.add_manifest_file(extracted, sha.new()).hexdigest()
 135                 if sha1 != required_digest:
 136                         raise BadDigest('Incorrect manifest -- archive is corrupted.\n'
 137                                         'Required digest: %s\n'
 138                                         'Actual digest: %s\n' %
 139                                         (required_digest, sha1))
 140
 141                 final_name = os.path.join(self.dir, required_digest)
 142                 if os.path.isdir(final_name):
 143                         raise Exception("Item %s already stored." % final_name)
 144                 if extract:
 145                         os.rename(os.path.join(tmp, extract), final_name)
 146                         os.rmdir(tmp)
 147                 else:
 148                         os.rename(tmp, final_name)
 149
 150         def extract(self, stream, command):
 151                 tmp = mkdtemp(dir = self.dir, prefix = 'tmp-')
 152                 try:
 153                         child = os.fork()
 154                         if child == 0:
 155                                 try:
 156                                         try:
 157                                                 os.chdir(tmp)
 158                                                 stream.seek(0)
 159                                                 os.dup2(stream.fileno(), 0)
 160                                                 os.execvp(command[0], command)
 161                                         except:
 162                                                 traceback.print_exc()
 163                                 finally:
 164                                         os._exit(1)
 165                         id, status = os.waitpid(child, 0)
 166                         assert id == child
 167                         if status != 0:
 168                                 raise Exception('Failed to extract archive; exit code %d' % status)
 169                 except:
 170                         shutil.rmtree(tmp)
 171                         raise
 172                 return tmp
 173
 174 class Stores(object):
 175         __slots__ = ['stores']
 176
 177         def __init__(self):
 178                 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
 179                 self.stores = [Store(user_store)]
 180
 181                 impl_dirs = basedir.load_first_config('0install.net', 'injector',
 182                                                           'implementation-dirs')
 183                 debug("Location of 'implementation-dirs' config file being used: '%s'", impl_dirs)
 184                 if impl_dirs:
 185                         for directory in file(impl_dirs):
 186                                 directory = directory.strip()
 187                                 if os.path.isdir(directory):
 188                                         self.stores.append(Store(directory))
 189                                         debug("Added system store '%s'", directory)
 190                                 else:
 191                                         info("Ignoring non-directory store '%s'", directory)
 192
 193         def lookup(self, digest):
 194                 """Search for digest in all stores."""
 195                 assert digest
 196                 if '/' in digest or '=' not in digest:
 197                         raise BadDigest('Syntax error in digest (use ALG=VALUE)')
 198                 for store in self.stores:
 199                         path = store.lookup(digest)
 200                         if path:
 201                                 return path
 202                 raise NotStored("Item with digest '%s' not found in stores. Searched:\n- %s" %
 203                         (digest, '\n- '.join([s.dir for s in self.stores])))
 204
 205         def add_dir_to_cache(self, required_digest, dir):
 206                 self.stores[0].add_dir_to_cache(required_digest, dir)
 207
 208         def add_archive_to_cache(self, required_digest, data, url, extract = None):
 209                 self.stores[0].add_archive_to_cache(required_digest, data, url, extract)