zeroinstall/injector/distro.py

   1 """
   2 Integration with native distribution package managers.
   3 @since: 0.28
   4 """
   5
   6 # Copyright (C) 2009, Thomas Leonard
   7 # See the README file for details, or visit http://0install.net.
   8
   9 from zeroinstall import _
  10 import os, platform, re, subprocess, sys
  11 from logging import warn, info
  12 from zeroinstall.injector import namespaces, model, arch
  13 from zeroinstall.support import basedir
  14
  15 _dotted_ints = '[0-9]+(?:\.[0-9]+)*'
  16
  17 # This matches a version number that would be a valid Zero Install version without modification
  18 _zeroinstall_regexp = '(?:%s)(?:-(?:pre|rc|post|)(?:%s))*' % (_dotted_ints, _dotted_ints)
  19
  20 # This matches the interesting bits of distribution version numbers
  21 _version_regexp = '(%s)(-r%s)?' % (_zeroinstall_regexp, _dotted_ints)
  22
  23 # We try to do updates atomically without locking, but we don't worry too much about
  24 # duplicate entries or being a little out of sync with the on-disk copy.
  25 class Cache(object):
  26         def __init__(self, cache_leaf, source, format):
  27                 """Maintain a cache file (e.g. ~/.cache/0install.net/injector/$name).
  28                 If the size or mtime of $source has changed, or the cache
  29                 format version if different, reset the cache first."""
  30                 self.cache_leaf = cache_leaf
  31                 self.source = source
  32                 self.format = format
  33                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
  34                                                          namespaces.config_prog)
  35                 self.cached_for = {}            # Attributes of source when cache was created
  36                 try:
  37                         self._load_cache()
  38                 except Exception, ex:
  39                         info(_("Failed to load cache (%s). Flushing..."), ex)
  40                         self.flush()
  41
  42         def flush(self):
  43                 # Wipe the cache
  44                 try:
  45                         info = os.stat(self.source)
  46                         mtime = int(info.st_mtime)
  47                         size = info.st_size
  48                 except Exception, ex:
  49                         warn("Failed to stat %s: %s", self.source, ex)
  50                         mtime = size = 0
  51                 self.cache = {}
  52                 import tempfile
  53                 tmp, tmp_name = tempfile.mkstemp(dir = self.cache_dir)
  54                 data = "mtime=%d\nsize=%d\nformat=%d\n\n" % (mtime, size, self.format)
  55                 while data:
  56                         wrote = os.write(tmp, data)
  57                         data = data[wrote:]
  58                 os.rename(tmp_name, os.path.join(self.cache_dir, self.cache_leaf))
  59
  60                 self._load_cache()
  61
  62         # Populate self.cache from our saved cache file.
  63         # Throws an exception if the cache doesn't exist or has the wrong format.
  64         def _load_cache(self):
  65                 self.cache = cache = {}
  66                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
  67                 try:
  68                         for line in stream:
  69                                 line = line.strip()
  70                                 if not line:
  71                                         break
  72                                 key, value = line.split('=', 1)
  73                                 if key in ('mtime', 'size', 'format'):
  74                                         self.cached_for[key] = int(value)
  75
  76                         self._check_valid()
  77
  78                         for line in stream:
  79                                 key, value = line.split('=', 1)
  80                                 cache[key] = value[:-1]
  81                 finally:
  82                         stream.close()
  83
  84         # Check the source file hasn't changed since we created the cache
  85         def _check_valid(self):
  86                 info = os.stat(self.source)
  87                 if self.cached_for['mtime'] != int(info.st_mtime):
  88                         raise Exception("Modification time of %s has changed" % self.source)
  89                 if self.cached_for['size'] != info.st_size:
  90                         raise Exception("Size of %s has changed" % self.source)
  91                 if self.cached_for.get('format', None) != self.format:
  92                         raise Exception("Format of cache has changed")
  93
  94         def get(self, key):
  95                 try:
  96                         self._check_valid()
  97                 except Exception, ex:
  98                         info(_("Cache needs to be refreshed: %s"), ex)
  99                         self.flush()
 100                         return None
 101                 else:
 102                         return self.cache.get(key, None)
 103
 104         def put(self, key, value):
 105                 cache_path = os.path.join(self.cache_dir, self.cache_leaf)
 106                 self.cache[key] = value
 107                 try:
 108                         stream = file(cache_path, 'a')
 109                         try:
 110                                 stream.write('%s=%s\n' % (key, value))
 111                         finally:
 112                                 stream.close()
 113                 except Exception, ex:
 114                         warn("Failed to write to cache %s: %s=%s: %s", cache_path, key, value, ex)
 115
 116 def try_cleanup_distro_version(version):
 117         """Try to turn a distribution version string into one readable by Zero Install.
 118         We do this by stripping off anything we can't parse.
 119         @return: the part we understood, or None if we couldn't parse anything
 120         @rtype: str"""
 121         if ':' in version:
 122                 version = version.split(':')[1] # Skip 'epoch'
 123         version = version.replace('_', '-')
 124         match = re.match(_version_regexp, version)
 125         if match:
 126                 version, revision = match.groups()
 127                 if revision is None:
 128                         return version
 129                 else:
 130                         return '%s-%s' % (version, revision[2:])
 131         return None
 132
 133 class Distribution(object):
 134         """Represents a distribution with which we can integrate.
 135         Sub-classes should specialise this to integrate with the package managers of
 136         particular distributions. This base class ignores the native package manager.
 137         @since: 0.28
 138         """
 139         _packagekit = None
 140
 141         def get_package_info(self, package, factory):
 142                 """Get information about the given package.
 143                 Add zero or more implementations using the factory (typically at most two
 144                 will be added; the currently installed version and the latest available).
 145                 @param package: package name (e.g. "gimp")
 146                 @type package: str
 147                 @param factory: function for creating new DistributionImplementation objects from IDs
 148                 @type factory: str -> L{model.DistributionImplementation}
 149                 """
 150                 return
 151
 152         def get_score(self, distribution):
 153                 """Indicate how closely the host distribution matches this one.
 154                 The <package-implementation> with the highest score is passed
 155                 to L{Distribution.get_package_info}. If several elements get
 156                 the same score, get_package_info is called for all of them.
 157                 @param distribution: a distribution name
 158                 @type distribution: str
 159                 @return: an integer, or None if there is no match at all
 160                 @rtype: int | None
 161                 """
 162                 return 0
 163
 164         def get_feed(self, master_feed):
 165                 """Generate a feed containing information about distribution packages.
 166                 This should immediately return a feed containing an implementation for the
 167                 package if it's already installed. Information about versions that could be
 168                 installed using the distribution's package manager can be added asynchronously
 169                 later (see L{fetch_candidates}).
 170                 @param master_feed: feed containing the <package-implementation> elements
 171                 @type master_feed: L{model.ZeroInstallFeed}
 172                 @rtype: L{model.ZeroInstallFeed}"""
 173
 174                 feed = model.ZeroInstallFeed(None)
 175                 feed.url = 'distribution:' + master_feed.url
 176
 177                 for item, item_attrs in master_feed.get_package_impls(self):
 178                         package = item_attrs.get('package', None)
 179                         if package is None:
 180                                 raise model.InvalidInterface(_("Missing 'package' attribute on %s") % item)
 181
 182                         def factory(id, only_if_missing = False, installed = True):
 183                                 assert id.startswith('package:')
 184                                 if id in feed.implementations:
 185                                         if only_if_missing:
 186                                                 return None
 187                                         warn(_("Duplicate ID '%s' for DistributionImplementation"), id)
 188                                 impl = model.DistributionImplementation(feed, id, self)
 189                                 feed.implementations[id] = impl
 190
 191                                 impl.installed = installed
 192                                 impl.metadata = item_attrs
 193
 194                                 item_main = item_attrs.get('main', None)
 195                                 if item_main and not item_main.startswith('/'):
 196                                         raise model.InvalidInterface(_("'main' attribute must be absolute, but '%s' doesn't start with '/'!") %
 197                                                                 item_main)
 198                                 impl.main = item_main
 199                                 impl.upstream_stability = model.packaged
 200
 201                                 return impl
 202
 203                         self.get_package_info(package, factory)
 204                 return feed
 205
 206         def fetch_candidates(self, master_feed):
 207                 """Collect information about versions we could install using
 208                 the distribution's package manager. On success, the distribution
 209                 feed in iface_cache is updated.
 210                 @return: a L{tasks.Blocker} if the task is in progress, or None if not"""
 211                 if self.packagekit.available:
 212                         package_names = [item.getAttribute("package") for item, item_attrs in master_feed.get_package_impls(self)]
 213                         return self.packagekit.fetch_candidates(package_names)
 214
 215         @property
 216         def packagekit(self):
 217                 """For use by subclasses.
 218                 @rtype: L{packagekit.PackageKit}"""
 219                 if not self._packagekit:
 220                         from zeroinstall.injector import packagekit
 221                         self._packagekit = packagekit.PackageKit()
 222                 return self._packagekit
 223
 224 class CachedDistribution(Distribution):
 225         """For distributions where querying the package database is slow (e.g. requires running
 226         an external command), we cache the results.
 227         @since: 0.39
 228         @deprecated: use Cache instead
 229         """
 230
 231         def __init__(self, db_status_file):
 232                 """@param db_status_file: update the cache when the timestamp of this file changes"""
 233                 self._status_details = os.stat(db_status_file)
 234
 235                 self.versions = {}
 236                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
 237                                                          namespaces.config_prog)
 238
 239                 try:
 240                         self._load_cache()
 241                 except Exception, ex:
 242                         info(_("Failed to load distribution database cache (%s). Regenerating..."), ex)
 243                         try:
 244                                 self.generate_cache()
 245                                 self._load_cache()
 246                         except Exception, ex:
 247                                 warn(_("Failed to regenerate distribution database cache: %s"), ex)
 248
 249         def _load_cache(self):
 250                 """Load {cache_leaf} cache file into self.versions if it is available and up-to-date.
 251                 Throws an exception if the cache should be (re)created."""
 252                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
 253
 254                 cache_version = None
 255                 for line in stream:
 256                         if line == '\n':
 257                                 break
 258                         name, value = line.split(': ')
 259                         if name == 'mtime' and int(value) != int(self._status_details.st_mtime):
 260                                 raise Exception(_("Modification time of package database file has changed"))
 261                         if name == 'size' and int(value) != self._status_details.st_size:
 262                                 raise Exception(_("Size of package database file has changed"))
 263                         if name == 'version':
 264                                 cache_version = int(value)
 265                 else:
 266                         raise Exception(_('Invalid cache format (bad header)'))
 267
 268                 if cache_version is None:
 269                         raise Exception(_('Old cache format'))
 270
 271                 versions = self.versions
 272                 for line in stream:
 273                         package, version, zi_arch = line[:-1].split('\t')
 274                         versionarch = (version, intern(zi_arch))
 275                         if package not in versions:
 276                                 versions[package] = [versionarch]
 277                         else:
 278                                 versions[package].append(versionarch)
 279
 280         def _write_cache(self, cache):
 281                 #cache.sort()   # Might be useful later; currently we don't care
 282                 import tempfile
 283                 fd, tmpname = tempfile.mkstemp(prefix = 'zeroinstall-cache-tmp',
 284                                                dir = self.cache_dir)
 285                 try:
 286                         stream = os.fdopen(fd, 'wb')
 287                         stream.write('version: 2\n')
 288                         stream.write('mtime: %d\n' % int(self._status_details.st_mtime))
 289                         stream.write('size: %d\n' % self._status_details.st_size)
 290                         stream.write('\n')
 291                         for line in cache:
 292                                 stream.write(line + '\n')
 293                         stream.close()
 294
 295                         os.rename(tmpname,
 296                                   os.path.join(self.cache_dir,
 297                                                self.cache_leaf))
 298                 except:
 299                         os.unlink(tmpname)
 300                         raise
 301
 302 # Maps machine type names used in packages to their Zero Install versions
 303 _canonical_machine = {
 304         'all' : '*',
 305         'any' : '*',
 306         'noarch' : '*',
 307         '(none)' : '*',
 308         'amd64': 'x86_64',
 309         'i386': 'i386',
 310         'i486': 'i486',
 311         'i586': 'i586',
 312         'i686': 'i686',
 313         'ppc64': 'ppc64',
 314         'ppc': 'ppc',
 315 }
 316
 317 host_machine = arch.canonicalize_machine(platform.uname()[4])
 318 def canonical_machine(package_machine):
 319         machine = _canonical_machine.get(package_machine, None)
 320         if machine is None:
 321                 # Safe default if we can't understand the arch
 322                 return host_machine
 323         return machine
 324
 325 class DebianDistribution(Distribution):
 326         """A dpkg-based distribution."""
 327
 328         cache_leaf = 'dpkg-status.cache'
 329
 330         def __init__(self, dpkg_status, pkgcache):
 331                 self.dpkg_cache = Cache('dpkg-status.cache', dpkg_status, 2)
 332                 self.apt_cache = {}
 333
 334         def _query_installed_package(self, package):
 335                 null = os.open('/dev/null', os.O_WRONLY)
 336                 child = subprocess.Popen(["dpkg-query", "-W", "--showformat=${Version}\t${Architecture}\t${Status}\n", "--", package],
 337                                                 stdout = subprocess.PIPE, stderr = null)
 338                 os.close(null)
 339                 stdout, stderr = child.communicate()
 340                 child.wait()
 341                 for line in stdout.split('\n'):
 342                         if not line: continue
 343                         version, debarch, status = line.split('\t', 2)
 344                         if not status.endswith(' installed'): continue
 345                         clean_version = try_cleanup_distro_version(version)
 346                         if clean_version:
 347                                 return '%s\t%s' % (clean_version, canonical_machine(debarch.strip()))
 348                         else:
 349                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 350
 351                 return '-'
 352
 353         def get_package_info(self, package, factory):
 354                 # Add any already-installed package...
 355                 installed_cached_info = self._get_dpkg_info(package)
 356
 357                 if installed_cached_info != '-':
 358                         installed_version, machine = installed_cached_info.split('\t')
 359                         impl = factory('package:deb:%s:%s:%s' % (package, installed_version, machine))
 360                         impl.version = model.parse_version(installed_version)
 361                         if machine != '*':
 362                                 impl.machine = machine
 363                 else:
 364                         installed_version = None
 365
 366                 # Add any uninstalled candidates (note: only one of these two methods will add anything)
 367
 368                 # From PackageKit...
 369                 self.packagekit.get_candidates(package, factory, 'package:deb')
 370
 371                 # From apt-cache...
 372                 cached = self.apt_cache.get(package, None)
 373                 if cached:
 374                         candidate_version = cached['version']
 375                         candidate_arch = cached['arch']
 376                         if candidate_version and candidate_version != installed_version:
 377                                 impl = factory('package:deb:%s:%s:%s' % (package, candidate_version, candidate_arch), installed = False)
 378                                 impl.version = model.parse_version(candidate_version)
 379                                 if candidate_arch != '*':
 380                                         impl.machine = candidate_arch
 381                                 def install(handler):
 382                                         raise model.SafeException(_("This program depends on '%s', which is a package that is available through your distribution. "
 383                                                         "Please install it manually using your distribution's tools and try again.") % package)
 384                                 impl.download_sources.append(model.DistributionSource(package, cached['size'], install, needs_confirmation = False))
 385
 386         def get_score(self, disto_name):
 387                 return int(disto_name == 'Debian')
 388
 389         def _get_dpkg_info(self, package):
 390                 installed_cached_info = self.dpkg_cache.get(package)
 391                 if installed_cached_info == None:
 392                         installed_cached_info = self._query_installed_package(package)
 393                         self.dpkg_cache.put(package, installed_cached_info)
 394
 395                 return installed_cached_info
 396
 397         def fetch_candidates(self, master_feed):
 398                 package_names = [item.getAttribute("package") for item, item_attrs in master_feed.get_package_impls(self)]
 399
 400                 if self.packagekit.available:
 401                         return self.packagekit.fetch_candidates(package_names)
 402
 403                 # No PackageKit. Use apt-cache directly.
 404                 for package in package_names:
 405                         # Check to see whether we could get a newer version using apt-get
 406                         try:
 407                                 null = os.open('/dev/null', os.O_WRONLY)
 408                                 child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null)
 409                                 os.close(null)
 410
 411                                 arch = version = size = None
 412                                 for line in child.stdout:
 413                                         line = line.strip()
 414                                         if line.startswith('Version: '):
 415                                                 version = line[9:]
 416                                                 version = try_cleanup_distro_version(version)
 417                                         elif line.startswith('Architecture: '):
 418                                                 arch = canonical_machine(line[14:].strip())
 419                                         elif line.startswith('Size: '):
 420                                                 size = int(line[6:].strip())
 421                                 if version and arch:
 422                                         cached = {'version': version, 'arch': arch, 'size': size}
 423                                 else:
 424                                         cached = None
 425                                 child.wait()
 426                         except Exception, ex:
 427                                 warn("'apt-cache show %s' failed: %s", package, ex)
 428                                 cached = None
 429                         # (multi-arch support? can there be multiple candidates?)
 430                         self.apt_cache[package] = cached
 431
 432 class RPMDistribution(CachedDistribution):
 433         """An RPM-based distribution."""
 434
 435         cache_leaf = 'rpm-status.cache'
 436
 437         def generate_cache(self):
 438                 cache = []
 439
 440                 for line in os.popen("rpm -qa --qf='%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\n'"):
 441                         package, version, rpmarch = line.split('\t', 2)
 442                         if package == 'gpg-pubkey':
 443                                 continue
 444                         zi_arch = canonical_machine(rpmarch.strip())
 445                         clean_version = try_cleanup_distro_version(version)
 446                         if clean_version:
 447                                 cache.append('%s\t%s\t%s' % (package, clean_version, zi_arch))
 448                         else:
 449                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 450
 451                 self._write_cache(cache)
 452
 453         def get_package_info(self, package, factory):
 454                 # Add installed versions...
 455                 versions = self.versions.get(package, [])
 456
 457                 for version, machine in versions:
 458                         impl = factory('package:rpm:%s:%s:%s' % (package, version, machine))
 459                         impl.version = model.parse_version(version)
 460                         if machine != '*':
 461                                 impl.machine = machine
 462
 463                 # Add any uninstalled candidates found by PackageKit
 464                 self.packagekit.get_candidates(package, factory, 'package:rpm')
 465
 466         def get_score(self, disto_name):
 467                 return int(disto_name == 'RPM')
 468
 469 class SlackDistribution(Distribution):
 470         """A Slack-based distribution."""
 471
 472         def __init__(self, packages_dir):
 473                 self._packages_dir = packages_dir
 474
 475         def get_package_info(self, package, factory):
 476                 # Add installed versions...
 477                 for entry in os.listdir(self._packages_dir):
 478                         name, version, arch, build = entry.rsplit('-', 3)
 479                         if name == package:
 480                                 zi_arch = canonical_machine(arch)
 481                                 clean_version = try_cleanup_distro_version("%s-%s" % (version, build))
 482                                 if not clean_version:
 483                                         warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': name})
 484                                         continue
 485
 486                                 impl = factory('package:slack:%s:%s:%s' % \
 487                                                 (package, clean_version, zi_arch))
 488                                 impl.version = model.parse_version(clean_version)
 489                                 if zi_arch != '*':
 490                                         impl.machine = zi_arch
 491
 492                 # Add any uninstalled candidates found by PackageKit
 493                 self.packagekit.get_candidates(package, factory, 'package:slack')
 494
 495         def get_score(self, disto_name):
 496                 return int(disto_name == 'Slack')
 497
 498 class GentooDistribution(Distribution):
 499
 500         def __init__(self, pkgdir):
 501                 self._pkgdir = pkgdir
 502
 503         def get_package_info(self, package, factory):
 504                 # Add installed versions...
 505                 _version_start_reqexp = '-[0-9]'
 506
 507                 if package.count('/') != 1: return
 508
 509                 category, leafname = package.split('/')
 510                 category_dir = os.path.join(self._pkgdir, category)
 511                 match_prefix = leafname + '-'
 512
 513                 if not os.path.isdir(category_dir): return
 514
 515                 for filename in os.listdir(category_dir):
 516                         if filename.startswith(match_prefix) and filename[len(match_prefix)].isdigit():
 517                                 name = file(os.path.join(category_dir, filename, 'PF')).readline().strip()
 518
 519                                 match = re.search(_version_start_reqexp, name)
 520                                 if match is None:
 521                                         warn(_('Cannot parse version from Gentoo package named "%(name)s"'), {'name': name})
 522                                         continue
 523                                 else:
 524                                         version = try_cleanup_distro_version(name[match.start() + 1:])
 525
 526                                 if category == 'app-emulation' and name.startswith('emul-'):
 527                                         __, __, machine, __ = name.split('-', 3)
 528                                 else:
 529                                         machine, __ = file(os.path.join(category_dir, filename, 'CHOST')).readline().split('-', 1)
 530                                 machine = arch.canonicalize_machine(machine)
 531
 532                                 impl = factory('package:gentoo:%s:%s:%s' % \
 533                                                 (package, version, machine))
 534                                 impl.version = model.parse_version(version)
 535                                 impl.machine = machine
 536
 537                 # Add any uninstalled candidates found by PackageKit
 538                 self.packagekit.get_candidates(package, factory, 'package:gentoo')
 539
 540         def get_score(self, disto_name):
 541                 return int(disto_name == 'Gentoo')
 542
 543 class PortsDistribution(Distribution):
 544
 545         def __init__(self, pkgdir):
 546                 self._pkgdir = pkgdir
 547
 548         def get_package_info(self, package, factory):
 549                 _name_version_regexp = '^(.+)-([^-]+)$'
 550
 551                 nameversion = re.compile(_name_version_regexp)
 552                 for pkgname in os.listdir(self._pkgdir):
 553                         pkgdir = os.path.join(self._pkgdir, pkgname)
 554                         if not os.path.isdir(pkgdir): continue
 555
 556                         #contents = file(os.path.join(pkgdir, '+CONTENTS')).readline().strip()
 557
 558                         match = nameversion.search(pkgname)
 559                         if match is None:
 560                                 warn(_('Cannot parse version from Ports package named "%(pkgname)s"'), {'pkgname': pkgname})
 561                                 continue
 562                         else:
 563                                 name = match.group(1)
 564                                 if name != package:
 565                                         continue
 566                                 version = try_cleanup_distro_version(match.group(2))
 567
 568                         machine = host_machine
 569
 570                         impl = factory('package:ports:%s:%s:%s' % \
 571                                                 (package, version, machine))
 572                         impl.version = model.parse_version(version)
 573                         impl.machine = machine
 574
 575         def get_score(self, disto_name):
 576                 return int(disto_name == 'Ports')
 577
 578 _host_distribution = None
 579 def get_host_distribution():
 580         """Get a Distribution suitable for the host operating system.
 581         Calling this twice will return the same object.
 582         @rtype: L{Distribution}"""
 583         global _host_distribution
 584         if not _host_distribution:
 585                 dpkg_db_status = '/var/lib/dpkg/status'
 586                 pkgcache = '/var/cache/apt/pkgcache.bin'
 587                 _rpm_db = '/var/lib/rpm/Packages'
 588                 _slack_db = '/var/log/packages'
 589                 _pkg_db = '/var/db/pkg'
 590
 591                 if os.path.isdir(_pkg_db):
 592                         if sys.platform.startswith("linux"):
 593                                 _host_distribution = GentooDistribution(_pkg_db)
 594                         elif sys.platform.startswith("freebsd"):
 595                                 _host_distribution = PortsDistribution(_pkg_db)
 596                 elif os.access(dpkg_db_status, os.R_OK):
 597                         _host_distribution = DebianDistribution(dpkg_db_status, pkgcache)
 598                 elif os.path.isfile(_rpm_db):
 599                         _host_distribution = RPMDistribution(_rpm_db)
 600                 elif os.path.isdir(_slack_db):
 601                         _host_distribution = SlackDistribution(_slack_db)
 602                 else:
 603                         _host_distribution = Distribution()
 604
 605         return _host_distribution