zeroinstall/injector/distro.py

   1 """
   2 Integration with native distribution package managers.
   3 @since: 0.28
   4 """
   5
   6 # Copyright (C) 2009, Thomas Leonard
   7 # See the README file for details, or visit http://0install.net.
   8
   9 from zeroinstall import _
  10 import os, re, glob, subprocess, sys
  11 from logging import warn, info
  12 from zeroinstall.injector import namespaces, model, arch
  13 from zeroinstall.support import basedir
  14
  15 _dotted_ints = '[0-9]+(?:\.[0-9]+)*'
  16
  17 # This matches a version number that would be a valid Zero Install version without modification
  18 _zeroinstall_regexp = '(?:%s)(?:-(?:pre|rc|post|)(?:%s))*' % (_dotted_ints, _dotted_ints)
  19
  20 # This matches the interesting bits of distribution version numbers
  21 _version_regexp = '(%s)(-r%s)?' % (_zeroinstall_regexp, _dotted_ints)
  22
  23 # We try to do updates atomically without locking, but we don't worry too much about
  24 # duplicate entries or being a little out of sync with the on-disk copy.
  25 class Cache(object):
  26         def __init__(self, cache_leaf, source, format):
  27                 """Maintain a cache file (e.g. ~/.cache/0install.net/injector/$name).
  28                 If the size or mtime of $source has changed, or the cache
  29                 format version if different, reset the cache first."""
  30                 self.cache_leaf = cache_leaf
  31                 self.source = source
  32                 self.format = format
  33                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
  34                                                          namespaces.config_prog)
  35                 self.cached_for = {}            # Attributes of source when cache was created
  36                 try:
  37                         self._load_cache()
  38                 except Exception, ex:
  39                         info(_("Failed to load cache (%s). Flushing..."), ex)
  40                         self.flush()
  41
  42         def flush(self):
  43                 # Wipe the cache
  44                 try:
  45                         info = os.stat(self.source)
  46                         mtime = int(info.st_mtime)
  47                         size = info.st_size
  48                 except Exception, ex:
  49                         warn("Failed to stat %s: %s", self.source, ex)
  50                         mtime = size = 0
  51                 self.cache = {}
  52                 import tempfile
  53                 tmp, tmp_name = tempfile.mkstemp(dir = self.cache_dir)
  54                 data = "mtime=%d\nsize=%d\nformat=%d\n\n" % (mtime, size, self.format)
  55                 while data:
  56                         wrote = os.write(tmp, data)
  57                         data = data[wrote:]
  58                 os.rename(tmp_name, os.path.join(self.cache_dir, self.cache_leaf))
  59
  60                 self._load_cache()
  61
  62         # Populate self.cache from our saved cache file.
  63         # Throws an exception if the cache doesn't exist or has the wrong format.
  64         def _load_cache(self):
  65                 self.cache = cache = {}
  66                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
  67                 try:
  68                         meta = {}
  69                         cached_format = False
  70                         for line in stream:
  71                                 line = line.strip()
  72                                 if not line:
  73                                         break
  74                                 key, value = line.split('=', 1)
  75                                 if key in ('mtime', 'size', 'format'):
  76                                         self.cached_for[key] = int(value)
  77
  78                         self._check_valid()
  79
  80                         for line in stream:
  81                                 key, value = line.split('=', 1)
  82                                 cache[key] = value[:-1]
  83                 finally:
  84                         stream.close()
  85
  86         # Check the source file hasn't changed since we created the cache
  87         def _check_valid(self):
  88                 info = os.stat(self.source)
  89                 if self.cached_for['mtime'] != int(info.st_mtime):
  90                         raise Exception("Modification time of %s has changed" % self.source)
  91                 if self.cached_for['size'] != info.st_size:
  92                         raise Exception("Size of %s has changed" % self.source)
  93                 if self.cached_for.get('format', None) != self.format:
  94                         raise Exception("Format of cache has changed")
  95
  96         def get(self, key):
  97                 try:
  98                         self._check_valid()
  99                 except Exception, ex:
 100                         info(_("Cache needs to be refreshed: %s"), ex)
 101                         self.flush()
 102                         return None
 103                 else:
 104                         return self.cache.get(key, None)
 105
 106         def put(self, key, value):
 107                 cache_path = os.path.join(self.cache_dir, self.cache_leaf)
 108                 self.cache[key] = value
 109                 try:
 110                         stream = file(cache_path, 'a')
 111                         try:
 112                                 stream.write('%s=%s\n' % (key, value))
 113                         finally:
 114                                 stream.close()
 115                 except Exception, ex:
 116                         warn("Failed to write to cache %s: %s=%s: %s", cache_path, key, value, ex)
 117
 118 def try_cleanup_distro_version(version):
 119         """Try to turn a distribution version string into one readable by Zero Install.
 120         We do this by stripping off anything we can't parse.
 121         @return: the part we understood, or None if we couldn't parse anything
 122         @rtype: str"""
 123         match = re.match(_version_regexp, version)
 124         if match:
 125                 version, revision = match.groups()
 126                 if revision is None:
 127                         return version
 128                 else:
 129                         return '%s-%s' % (version, revision[2:])
 130         return None
 131
 132 class Distribution(object):
 133         """Represents a distribution with which we can integrate.
 134         Sub-classes should specialise this to integrate with the package managers of
 135         particular distributions. This base class ignores the native package manager.
 136         @since: 0.28
 137         """
 138
 139         def get_package_info(self, package, factory):
 140                 """Get information about the given package.
 141                 Add zero or more implementations using the factory (typically at most two
 142                 will be added; the currently installed version and the latest available).
 143                 @param package: package name (e.g. "gimp")
 144                 @type package: str
 145                 @param factory: function for creating new DistributionImplementation objects from IDs
 146                 @type factory: str -> L{model.DistributionImplementation}
 147                 """
 148                 return
 149
 150         def get_score(self, distribution):
 151                 """Indicate how closely the host distribution matches this one.
 152                 The <package-implementation> with the highest score is passed
 153                 to L{Distribution.get_package_info}. If several elements get
 154                 the same score, get_package_info is called for all of them.
 155                 @param distribution: a distribution name
 156                 @type distribution: str
 157                 @return: an integer, or None if there is no match at all
 158                 @rtype: int | None
 159                 """
 160                 return 0
 161
 162         def get_installed(self, package_id):
 163                 """Check whether 'package' is currently installed.
 164                 @param package_id: the Implementation ID used by get_package_info
 165                 @type package_id: str
 166                 @return: True iff the package is currently installed"""
 167                 return True
 168
 169 class CachedDistribution(Distribution):
 170         """For distributions where querying the package database is slow (e.g. requires running
 171         an external command), we cache the results.
 172         @since: 0.39
 173         @deprecated: use Cache instead
 174         """
 175
 176         def __init__(self, db_status_file):
 177                 """@param db_status_file: update the cache when the timestamp of this file changes"""
 178                 self._status_details = os.stat(db_status_file)
 179
 180                 self.versions = {}
 181                 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
 182                                                          namespaces.config_prog)
 183
 184                 try:
 185                         self._load_cache()
 186                 except Exception, ex:
 187                         info(_("Failed to load distribution database cache (%s). Regenerating..."), ex)
 188                         try:
 189                                 self.generate_cache()
 190                                 self._load_cache()
 191                         except Exception, ex:
 192                                 warn(_("Failed to regenerate distribution database cache: %s"), ex)
 193
 194         def _load_cache(self):
 195                 """Load {cache_leaf} cache file into self.versions if it is available and up-to-date.
 196                 Throws an exception if the cache should be (re)created."""
 197                 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
 198
 199                 cache_version = None
 200                 for line in stream:
 201                         if line == '\n':
 202                                 break
 203                         name, value = line.split(': ')
 204                         if name == 'mtime' and int(value) != int(self._status_details.st_mtime):
 205                                 raise Exception(_("Modification time of package database file has changed"))
 206                         if name == 'size' and int(value) != self._status_details.st_size:
 207                                 raise Exception(_("Size of package database file has changed"))
 208                         if name == 'version':
 209                                 cache_version = int(value)
 210                 else:
 211                         raise Exception(_('Invalid cache format (bad header)'))
 212
 213                 if cache_version is None:
 214                         raise Exception(_('Old cache format'))
 215
 216                 versions = self.versions
 217                 for line in stream:
 218                         package, version, zi_arch = line[:-1].split('\t')
 219                         versionarch = (version, intern(zi_arch))
 220                         if package not in versions:
 221                                 versions[package] = [versionarch]
 222                         else:
 223                                 versions[package].append(versionarch)
 224
 225         def _write_cache(self, cache):
 226                 #cache.sort()   # Might be useful later; currently we don't care
 227                 import tempfile
 228                 fd, tmpname = tempfile.mkstemp(prefix = 'zeroinstall-cache-tmp',
 229                                                dir = self.cache_dir)
 230                 try:
 231                         stream = os.fdopen(fd, 'wb')
 232                         stream.write('version: 2\n')
 233                         stream.write('mtime: %d\n' % int(self._status_details.st_mtime))
 234                         stream.write('size: %d\n' % self._status_details.st_size)
 235                         stream.write('\n')
 236                         for line in cache:
 237                                 stream.write(line + '\n')
 238                         stream.close()
 239
 240                         os.rename(tmpname,
 241                                   os.path.join(self.cache_dir,
 242                                                self.cache_leaf))
 243                 except:
 244                         os.unlink(tmpname)
 245                         raise
 246
 247 # Maps machine type names used in packages to their Zero Install versions
 248 _canonical_machine = {
 249         'all' : '*',
 250         'any' : '*',
 251         'noarch' : '*',
 252         '(none)' : '*',
 253         'amd64': 'x86_64',
 254         'i386': 'i386',
 255         'i486': 'i486',
 256         'i586': 'i586',
 257         'i686': 'i686',
 258         'ppc64': 'ppc64',
 259         'ppc': 'ppc',
 260 }
 261
 262 host_machine = arch.canonicalize_machine(os.uname()[-1])
 263 def canonical_machine(package_machine):
 264         machine = _canonical_machine.get(package_machine, None)
 265         if machine is None:
 266                 # Safe default if we can't understand the arch
 267                 return host_machine
 268         return machine
 269
 270 class DebianDistribution(Distribution):
 271         """A dpkg-based distribution."""
 272
 273         cache_leaf = 'dpkg-status.cache'
 274
 275         def __init__(self, dpkg_status, pkgcache):
 276                 self.dpkg_cache = Cache('dpkg-status.cache', dpkg_status, 2)
 277                 self.apt_cache = Cache('apt-cache-cache', pkgcache, 3)
 278
 279         def _query_installed_package(self, package):
 280                 null = os.open('/dev/null', os.O_WRONLY)
 281                 child = subprocess.Popen(["dpkg-query", "-W", "--showformat=${Version}\t${Architecture}\t${Status}\n", "--", package],
 282                                                 stdout = subprocess.PIPE, stderr = null)
 283                 os.close(null)
 284                 stdout, stderr = child.communicate()
 285                 child.wait()
 286                 for line in stdout.split('\n'):
 287                         if not line: continue
 288                         version, debarch, status = line.split('\t', 2)
 289                         if not status.endswith(' installed'): continue
 290                         if ':' in version:
 291                                 # Debian's 'epoch' system
 292                                 version = version.split(':', 1)[1]
 293                         clean_version = try_cleanup_distro_version(version)
 294                         if clean_version:
 295                                 return '%s\t%s' % (clean_version, canonical_machine(debarch.strip()))
 296                         else:
 297                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 298
 299                 return '-'
 300
 301         def get_package_info(self, package, factory):
 302                 installed_cached_info = self._get_dpkg_info(package)
 303
 304                 if installed_cached_info != '-':
 305                         installed_version, machine = installed_cached_info.split('\t')
 306                         impl = factory('package:deb:%s:%s' % (package, installed_version))
 307                         impl.version = model.parse_version(installed_version)
 308                         if machine != '*':
 309                                 impl.machine = machine
 310                 else:
 311                         installed_version = None
 312
 313                 # Check to see whether we could get a newer version using apt-get
 314
 315                 cached = self.apt_cache.get(package)
 316                 if cached is None:
 317                         try:
 318                                 null = os.open('/dev/null', os.O_WRONLY)
 319                                 child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null)
 320                                 os.close(null)
 321
 322                                 arch = version = size = None
 323                                 for line in child.stdout:
 324                                         line = line.strip()
 325                                         if line.startswith('Version: '):
 326                                                 version = line[9:]
 327                                                 if ':' in version:
 328                                                         # Debian's 'epoch' system
 329                                                         version = version.split(':', 1)[1]
 330                                                 version = try_cleanup_distro_version(version)
 331                                         elif line.startswith('Architecture: '):
 332                                                 arch = canonical_machine(line[14:].strip())
 333                                         elif line.startswith('Size: '):
 334                                                 size = int(line[6:].strip())
 335                                 if version and arch:
 336                                         cached = '%s\t%s\t%d' % (version, arch, size)
 337                                 else:
 338                                         cached = '-'
 339                                 child.wait()
 340                         except Exception, ex:
 341                                 warn("'apt-cache show %s' failed: %s", package, ex)
 342                                 cached = '-'
 343                         # (multi-arch support? can there be multiple candidates?)
 344                         self.apt_cache.put(package, cached)
 345
 346                 if cached != '-':
 347                         candidate_version, candidate_arch, candidate_size = cached.split('\t')
 348                         if candidate_version and candidate_version != installed_version:
 349                                 impl = factory('package:deb:%s:%s' % (package, candidate_version))
 350                                 impl.version = model.parse_version(candidate_version)
 351                                 if candidate_arch != '*':
 352                                         impl.machine = candidate_arch
 353                                 impl.download_sources.append(model.DistributionSource(package, candidate_size))
 354
 355         def get_score(self, disto_name):
 356                 return int(disto_name == 'Debian')
 357
 358         def _get_dpkg_info(self, package):
 359                 installed_cached_info = self.dpkg_cache.get(package)
 360                 if installed_cached_info == None:
 361                         installed_cached_info = self._query_installed_package(package)
 362                         self.dpkg_cache.put(package, installed_cached_info)
 363
 364                 return installed_cached_info
 365
 366         def get_installed(self, package_id):
 367                 details = package_id.split(':', 3)
 368                 assert details[0] == 'package'
 369                 package = details[2]
 370                 info = self._get_dpkg_info(package)
 371                 if info is '-': return False
 372                 installed_version, machine = info.split('\t')
 373                 installed_id = 'package:deb:%s:%s' % (package, installed_version)
 374                 return package_id == installed_id
 375
 376 class RPMDistribution(CachedDistribution):
 377         """An RPM-based distribution."""
 378
 379         cache_leaf = 'rpm-status.cache'
 380
 381         def generate_cache(self):
 382                 cache = []
 383
 384                 for line in os.popen("rpm -qa --qf='%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\n'"):
 385                         package, version, rpmarch = line.split('\t', 2)
 386                         if package == 'gpg-pubkey':
 387                                 continue
 388                         zi_arch = canonical_machine(rpmarch.strip())
 389                         clean_version = try_cleanup_distro_version(version)
 390                         if clean_version:
 391                                 cache.append('%s\t%s\t%s' % (package, clean_version, zi_arch))
 392                         else:
 393                                 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
 394
 395                 self._write_cache(cache)
 396
 397         def get_package_info(self, package, factory):
 398                 try:
 399                         versions = self.versions[package]
 400                 except KeyError:
 401                         return
 402
 403                 for version, machine in versions:
 404                         impl = factory('package:rpm:%s:%s:%s' % (package, version, machine))
 405                         impl.version = model.parse_version(version)
 406                         if machine != '*':
 407                                 impl.machine = machine
 408
 409         def get_score(self, disto_name):
 410                 return int(disto_name == 'RPM')
 411
 412 class SlackDistribution(Distribution):
 413         """A Slack-based distribution."""
 414
 415         def __init__(self, packages_dir):
 416                 self._packages_dir = packages_dir
 417
 418         def get_package_info(self, package, factory):
 419                 for entry in os.listdir(self._packages_dir):
 420                         name, version, arch, build = entry.rsplit('-', 3)
 421                         if name == package:
 422                                 zi_arch = canonical_machine(arch)
 423                                 clean_version = try_cleanup_distro_version("%s-%s" % (version, build))
 424                                 if not clean_version:
 425                                         warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': name})
 426                                         continue
 427
 428                                 impl = factory('package:slack:%s:%s:%s' % \
 429                                                 (package, clean_version, zi_arch))
 430                                 impl.version = model.parse_version(clean_version)
 431                                 if zi_arch != '*':
 432                                         impl.machine = zi_arch
 433
 434         def get_score(self, disto_name):
 435                 return int(disto_name == 'Slack')
 436
 437 class GentooDistribution(Distribution):
 438
 439         def __init__(self, pkgdir):
 440                 self._pkgdir = pkgdir
 441
 442         def get_package_info(self, package, factory):
 443                 _version_start_reqexp = '-[0-9]'
 444
 445                 if package.count('/') != 1: return
 446
 447                 category, leafname = package.split('/')
 448                 category_dir = os.path.join(self._pkgdir, category)
 449                 match_prefix = leafname + '-'
 450
 451                 if not os.path.isdir(category_dir): return
 452
 453                 for filename in os.listdir(category_dir):
 454                         if filename.startswith(match_prefix) and filename[len(match_prefix)].isdigit():
 455                                 name = file(os.path.join(category_dir, filename, 'PF')).readline().strip()
 456
 457                                 match = re.search(_version_start_reqexp, name)
 458                                 if match is None:
 459                                         warn(_('Cannot parse version from Gentoo package named "%(name)s"'), {'name': name})
 460                                         continue
 461                                 else:
 462                                         version = try_cleanup_distro_version(name[match.start() + 1:])
 463
 464                                 if category == 'app-emulation' and name.startswith('emul-'):
 465                                         __, __, machine, __ = name.split('-', 3)
 466                                 else:
 467                                         machine, __ = file(os.path.join(category_dir, filename, 'CHOST')).readline().split('-', 1)
 468                                 machine = arch.canonicalize_machine(machine)
 469
 470                                 impl = factory('package:gentoo:%s:%s:%s' % \
 471                                                 (package, version, machine))
 472                                 impl.version = model.parse_version(version)
 473                                 impl.machine = machine
 474
 475         def get_score(self, disto_name):
 476                 return int(disto_name == 'Gentoo')
 477
 478 class PortsDistribution(Distribution):
 479
 480         def __init__(self, pkgdir):
 481                 self._pkgdir = pkgdir
 482
 483         def get_package_info(self, package, factory):
 484                 _version_start_reqexp = '-[0-9]'
 485
 486                 for pkgname in os.listdir(self._pkgdir):
 487                         pkgdir = os.path.join(self._pkgdir, pkgname)
 488                         if not os.path.isdir(pkgdir): continue
 489
 490                         #contents = file(os.path.join(pkgdir, '+CONTENTS')).readline().strip()
 491
 492                         match = re.search(_version_start_reqexp, pkgname)
 493                         if match is None:
 494                                 warn(_('Cannot parse version from Ports package named "%(pkgname)s"'), {'name': pkgname})
 495                                 continue
 496                         else:
 497                                 name = pkgname[0:match.start()]
 498                                 version = try_cleanup_distro_version(pkgname[match.start() + 1:])
 499
 500                         machine = host_machine
 501
 502                         impl = factory('package:ports:%s:%s:%s' % \
 503                                                 (package, version, machine))
 504                         impl.version = model.parse_version(version)
 505                         impl.machine = machine
 506
 507         def get_score(self, disto_name):
 508                 return int(disto_name == 'Ports')
 509
 510 _host_distribution = None
 511 def get_host_distribution():
 512         """Get a Distribution suitable for the host operating system.
 513         Calling this twice will return the same object.
 514         @rtype: L{Distribution}"""
 515         global _host_distribution
 516         if not _host_distribution:
 517                 dpkg_db_status = '/var/lib/dpkg/status'
 518                 pkgcache = '/var/cache/apt/pkgcache.bin'
 519                 _rpm_db = '/var/lib/rpm/Packages'
 520                 _slack_db = '/var/log/packages'
 521                 _pkg_db = '/var/db/pkg'
 522
 523                 if os.path.isdir(_pkg_db):
 524                         if sys.platform.startswith("linux"):
 525                                 _host_distribution = GentooDistribution(_pkg_db)
 526                         elif sys.platform.startswith("freebsd"):
 527                                 _host_distribution = PortsDistribution(_pkg_db)
 528                 elif os.access(dpkg_db_status, os.R_OK):
 529                         _host_distribution = DebianDistribution(dpkg_db_status, pkgcache)
 530                 elif os.path.isfile(_rpm_db):
 531                         _host_distribution = RPMDistribution(_rpm_db)
 532                 elif os.path.isdir(_slack_db):
 533                         _host_distribution = SlackDistribution(_slack_db)
 534                 else:
 535                         _host_distribution = Distribution()
 536
 537         return _host_distribution