zeroinstall/injector/distro.py

   1 """
   2 Integration with native distribution package managers.
   3 @since: 0.28
   4 """
   5
   6 # Copyright (C) 2009, Thomas Leonard
   7 # See the README file for details, or visit http://0install.net.
   8
   9 from zeroinstall import _
  10 import os, re, glob, subprocess
  11 from logging import warn, info
  12 from zeroinstall.injector import namespaces, model
  13 from zeroinstall.support import basedir
  14
  15 _dotted_ints = '[0-9]+(?:\.[0-9]+)*'
  16
  17 # This matches a version number that would be a valid Zero Install version without modification
  18 _zeroinstall_regexp = '(?:%s)(?:-(?:pre|rc|post|)(?:%s))*' % (_dotted_ints, _dotted_ints)
  19
  20 # This matches the interesting bits of distribution version numbers
  21 _version_regexp = '(%s)(-r%s)?' % (_zeroinstall_regexp, _dotted_ints)
  22
  23 # We try to do updates atomically without locking, but we don't worry too much about
  24 # duplicate entries or being a little out of sync with the on-disk copy.
  25 class Cache(object):
  26         def __init__(self, cache_leaf, source):
  27                 """Maintain a cache file (e.g. ~/.cache/0install.net/injector/$name).
  28                 If the size or mtime of $source has changed, reset the cache first."""
  29                 self.cache_leaf = cache_leaf
  30                 self.source = source
  31                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
  32                                                          namespaces.config_prog)
  33                 try:
  34                         self._load_cache()
  35                 except Exception, ex:
  36                         info(_("Failed to load cache (%s). Flushing..."), ex)
  37                         self.flush()
  38
  39         def flush(self):
  40                 try:
  41                         info = os.stat(self.source)
  42                         mtime = int(info.st_mtime)
  43                         size = info.st_size
  44                 except Exception, ex:
  45                         warn("Failed to stat %s: %s", self.source, ex)
  46                         mtime = size = 0
  47                 self.cache = {}
  48                 import tempfile
  49                 tmp, tmp_name = tempfile.mkstemp(dir = self.cache_dir)
  50                 data = "mtime=%d\nsize=%d\n\n" % (mtime, size)
  51                 while data:
  52                         wrote = os.write(tmp, data)
  53                         data = data[wrote:]
  54                 os.rename(tmp_name, os.path.join(self.cache_dir, self.cache_leaf))
  55
  56         # Populate self.cache from our saved cache file.
  57         # Throws an exception if the cache doesn't exist or has the wrong format.
  58         def _load_cache(self):
  59                 self.cache = cache = {}
  60                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
  61                 try:
  62                         info = os.stat(self.source)
  63                         meta = {}
  64                         for line in stream:
  65                                 line = line.strip()
  66                                 if not line:
  67                                         break
  68                                 key, value = line.split('=', 1)
  69                                 if key == 'mtime':
  70                                         if int(value) != int(info.st_mtime):
  71                                                 raise Exception("Modification time of %s has changed" % self.source)
  72                                 if key == 'size':
  73                                         if int(value) != info.st_size:
  74                                                 raise Exception("Size of %s has changed" % self.source)
  75
  76                         for line in stream:
  77                                 key, value = line.split('=', 1)
  78                                 cache[key] = value[:-1]
  79                 finally:
  80                         stream.close()
  81
  82         def get(self, key):
  83                 return self.cache.get(key, None)
  84
  85         def put(self, key, value):
  86                 cache_path = os.path.join(self.cache_dir, self.cache_leaf)
  87                 self.cache[key] = value
  88                 try:
  89                         stream = file(cache_path, 'a')
  90                         try:
  91                                 stream.write('%s=%s\n' % (key, value))
  92                         finally:
  93                                 stream.close()
  94                 except Exception, ex:
  95                         warn("Failed to write to cache %s: %s=%s: %s", cache_path, key, value, ex)
  96
  97 def try_cleanup_distro_version(version):
  98         """Try to turn a distribution version string into one readable by Zero Install.
  99         We do this by stripping off anything we can't parse.
 100         @return: the part we understood, or None if we couldn't parse anything
 101         @rtype: str"""
 102         match = re.match(_version_regexp, version)
 103         if match:
 104                 version, revision = match.groups()
 105                 if revision is None:
 106                         return version
 107                 else:
 108                         return '%s-%s' % (version, revision[2:])
 109         return None
 110
 111 class Distribution(object):
 112         """Represents a distribution with which we can integrate.
 113         Sub-classes should specialise this to integrate with the package managers of
 114         particular distributions. This base class ignores the native package manager.
 115         @since: 0.28
 116         """
 117
 118         def get_package_info(self, package, factory):
 119                 """Get information about the given package.
 120                 Add zero or more implementations using the factory (typically at most two
 121                 will be added; the currently installed version and the latest available).
 122                 @param package: package name (e.g. "gimp")
 123                 @type package: str
 124                 @param factory: function for creating new DistributionImplementation objects from IDs
 125                 @type factory: str -> L{model.DistributionImplementation}
 126                 """
 127                 return
 128
 129         def get_score(self, distribution):
 130                 """Indicate how closely the host distribution matches this one.
 131                 The <package-implementation> with the highest score is passed
 132                 to L{Distribution.get_package_info}. If several elements get
 133                 the same score, get_package_info is called for all of them.
 134                 @param distribution: a distribution name
 135                 @type distribution: str
 136                 @return: an integer, or None if there is no match at all
 137                 @rtype: int | None
 138                 """
 139                 return 0
 140
 141 class CachedDistribution(Distribution):
 142         """For distributions where querying the package database is slow (e.g. requires running
 143         an external command), we cache the results.
 144         @since: 0.39
 145         """
 146
 147         def __init__(self, db_status_file):
 148                 """@param db_status_file: update the cache when the timestamp of this file changes"""
 149                 self._status_details = os.stat(db_status_file)
 150
 151                 self.versions = {}
 152                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
 153                                                          namespaces.config_prog)
 154
 155                 try:
 156                         self._load_cache()
 157                 except Exception, ex:
 158                         info(_("Failed to load distribution database cache (%s). Regenerating..."), ex)
 159                         try:
 160                                 self.generate_cache()
 161                                 self._load_cache()
 162                         except Exception, ex:
 163                                 warn(_("Failed to regenerate distribution database cache: %s"), ex)
 164
 165         def _load_cache(self):
 166                 """Load {cache_leaf} cache file into self.versions if it is available and up-to-date.
 167                 Throws an exception if the cache should be (re)created."""
 168                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
 169
 170                 cache_version = None
 171                 for line in stream:
 172                         if line == '\n':
 173                                 break
 174                         name, value = line.split(': ')
 175                         if name == 'mtime' and int(value) != int(self._status_details.st_mtime):
 176                                 raise Exception(_("Modification time of package database file has changed"))
 177                         if name == 'size' and int(value) != self._status_details.st_size:
 178                                 raise Exception(_("Size of package database file has changed"))
 179                         if name == 'version':
 180                                 cache_version = int(value)
 181                 else:
 182                         raise Exception(_('Invalid cache format (bad header)'))
 183
 184                 if cache_version is None:
 185                         raise Exception(_('Old cache format'))
 186
 187                 versions = self.versions
 188                 for line in stream:
 189                         package, version, zi_arch = line[:-1].split('\t')
 190                         versionarch = (version, intern(zi_arch))
 191                         if package not in versions:
 192                                 versions[package] = [versionarch]
 193                         else:
 194                                 versions[package].append(versionarch)
 195
 196         def _write_cache(self, cache):
 197                 #cache.sort()   # Might be useful later; currently we don't care
 198                 import tempfile
 199                 fd, tmpname = tempfile.mkstemp(prefix = 'zeroinstall-cache-tmp',
 200                                                dir = self.cache_dir)
 201                 try:
 202                         stream = os.fdopen(fd, 'wb')
 203                         stream.write('version: 2\n')
 204                         stream.write('mtime: %d\n' % int(self._status_details.st_mtime))
 205                         stream.write('size: %d\n' % self._status_details.st_size)
 206                         stream.write('\n')
 207                         for line in cache:
 208                                 stream.write(line + '\n')
 209                         stream.close()
 210
 211                         os.rename(tmpname,
 212                                   os.path.join(self.cache_dir,
 213                                                self.cache_leaf))
 214                 except:
 215                         os.unlink(tmpname)
 216                         raise
 217
 218 # Maps machine type names used in packages to their Zero Install versions
 219 _canonical_machine = {
 220         'all' : '*',
 221         'any' : '*',
 222         'amd64': 'x86_64',
 223         'i386': 'i386',
 224 }
 225
 226 host_machine = os.uname()[-1]
 227 def canonical_machine(package_machine):
 228         machine = _canonical_machine.get(package_machine, None)
 229         if machine is None:
 230                 # Safe default if we can't understand the arch
 231                 return host_machine
 232         return machine
 233
 234 class DebianDistribution(CachedDistribution):
 235         """A dpkg-based distribution."""
 236
 237         cache_leaf = 'dpkg-status.cache'
 238
 239         def __init__(self, dpkg_status):
 240                 CachedDistribution.__init__(self, dpkg_status)
 241                 self.apt_cache = Cache('apt-cache-cache', '/var/cache/apt/pkgcache.bin')
 242
 243         def generate_cache(self):
 244                 cache = []
 245
 246                 for line in os.popen("dpkg-query -W --showformat='${Package}\t${Version}\t${Architecture}\n'"):
 247                         package, version, debarch = line.split('\t', 2)
 248                         if ':' in version:
 249                                 # Debian's 'epoch' system
 250                                 version = version.split(':', 1)[1]
 251                         clean_version = try_cleanup_distro_version(version)
 252                         if clean_version:
 253                                 cache.append('%s\t%s\t%s' % (package, clean_version, canonical_machine(debarch.strip())))
 254                         else:
 255                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 256
 257                 self._write_cache(cache)
 258
 259         def get_package_info(self, package, factory):
 260                 try:
 261                         installed_version, machine = self.versions[package][0]
 262                 except KeyError:
 263                         installed_version = None
 264                 else:
 265                         impl = factory('package:deb:%s:%s' % (package, installed_version))
 266                         impl.version = model.parse_version(installed_version)
 267                         if machine != '*':
 268                                 impl.machine = machine
 269
 270                 # Check to see whether we could get a newer version using apt-get
 271
 272                 cached = self.apt_cache.get(package)
 273                 if cached is None:
 274                         try:
 275                                 null = os.open('/dev/null', os.O_WRONLY)
 276                                 child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null)
 277                                 os.close(null)
 278
 279                                 arch = version = None
 280                                 for line in child.stdout:
 281                                         line = line.strip()
 282                                         if line.startswith('Version: '):
 283                                                 version = line[9:]
 284                                                 if ':' in version:
 285                                                         # Debian's 'epoch' system
 286                                                         version = version.split(':', 1)[1]
 287                                                 version = try_cleanup_distro_version(version)
 288                                         elif line.startswith('Architecture: '):
 289                                                 arch = canonical_machine(line[14:].strip())
 290                                 if version and arch:
 291                                         cached = '%s\t%s' % (version, arch)
 292                                 else:
 293                                         cached = '-'
 294                                 child.wait()
 295                         except Exception, ex:
 296                                 warn("'apt-cache show %s' failed: %s", package, ex)
 297                                 cached = '-'
 298                         # (multi-arch support? can there be multiple candidates?)
 299                         self.apt_cache.put(package, cached)
 300
 301                 if cached != '-':
 302                         candidate_version, candidate_arch = cached.split('\t')
 303                         if candidate_version and candidate_version != installed_version:
 304                                 impl = factory('package:deb:%s:%s' % (package, candidate_version))
 305                                 impl.version = model.parse_version(candidate_version)
 306                                 impl.installed = False
 307                                 if candidate_arch != '*':
 308                                         impl.machine = candidate_arch
 309
 310         def get_score(self, disto_name):
 311                 return int(disto_name == 'Debian')
 312
 313 class RPMDistribution(CachedDistribution):
 314         """An RPM-based distribution."""
 315
 316         cache_leaf = 'rpm-status.cache'
 317
 318         def generate_cache(self):
 319                 cache = []
 320
 321                 for line in os.popen("rpm -qa --qf='%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\n'"):
 322                         package, version, rpmarch = line.split('\t', 2)
 323                         if package == 'gpg-pubkey':
 324                                 continue
 325                         if rpmarch == 'amd64\n':
 326                                 zi_arch = 'x86_64'
 327                         elif rpmarch == 'noarch\n' or rpmarch == "(none)\n":
 328                                 zi_arch = '*'
 329                         else:
 330                                 zi_arch = rpmarch.strip()
 331                         clean_version = try_cleanup_distro_version(version)
 332                         if clean_version:
 333                                 cache.append('%s\t%s\t%s' % (package, clean_version, zi_arch))
 334                         else:
 335                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 336
 337                 self._write_cache(cache)
 338
 339         def get_package_info(self, package, factory):
 340                 try:
 341                         versions = self.versions[package]
 342                 except KeyError:
 343                         return
 344
 345                 for version, machine in versions:
 346                         impl = factory('package:rpm:%s:%s:%s' % (package, version, machine))
 347                         impl.version = model.parse_version(version)
 348                         if machine != '*':
 349                                 impl.machine = machine
 350
 351         def get_score(self, disto_name):
 352                 return int(disto_name == 'RPM')
 353
 354 class GentooDistribution(Distribution):
 355
 356         def __init__(self, pkgdir):
 357                 self._pkgdir = pkgdir
 358
 359         def get_package_info(self, package, factory):
 360                 _version_start_reqexp = '-[0-9]'
 361
 362                 if package.count('/') != 1: return
 363
 364                 category, leafname = package.split('/')
 365                 category_dir = os.path.join(self._pkgdir, category)
 366                 match_prefix = leafname + '-'
 367
 368                 if not os.path.isdir(category_dir): return
 369
 370                 for filename in os.listdir(category_dir):
 371                         if filename.startswith(match_prefix) and filename[len(match_prefix)].isdigit():
 372                                 name = file(os.path.join(category_dir, filename, 'PF')).readline().strip()
 373
 374                                 match = re.search(_version_start_reqexp, name)
 375                                 if match is None:
 376                                         warn(_('Cannot parse version from Gentoo package named "%(name)s"'), {'name': name})
 377                                         continue
 378                                 else:
 379                                         version = try_cleanup_distro_version(name[match.start() + 1:])
 380
 381                                 machine = file(os.path.join(category_dir, filename, 'CHOST')).readline().split('-')[0]
 382
 383                                 impl = factory('package:gentoo:%s:%s:%s' % \
 384                                                 (package, version, machine))
 385                                 impl.version = model.parse_version(version)
 386
 387         def get_score(self, disto_name):
 388                 return int(disto_name == 'Gentoo')
 389
 390
 391 _host_distribution = None
 392 def get_host_distribution():
 393         """Get a Distribution suitable for the host operating system.
 394         Calling this twice will return the same object.
 395         @rtype: L{Distribution}"""
 396         global _host_distribution
 397         if not _host_distribution:
 398                 _dpkg_db_status = '/var/lib/dpkg/status'
 399                 _rpm_db = '/var/lib/rpm/Packages'
 400                 _gentoo_db = '/var/db/pkg'
 401
 402                 if os.path.isdir(_gentoo_db):
 403                         _host_distribution = GentooDistribution(_gentoo_db)
 404                 elif os.access(_dpkg_db_status, os.R_OK):
 405                         _host_distribution = DebianDistribution(_dpkg_db_status)
 406                 elif os.path.isfile(_rpm_db):
 407                         _host_distribution = RPMDistribution(_rpm_db)
 408                 else:
 409                         _host_distribution = Distribution()
 410
 411         return _host_distribution