Updated exception handling to Python 2.6 syntax
[zeroinstall.git] / zeroinstall / injector / distro.py
blob80a7c031af12b0679c2809b98b8772240a1c5b94
1 """
2 Integration with native distribution package managers.
3 @since: 0.28
4 """
6 # Copyright (C) 2009, Thomas Leonard
7 # See the README file for details, or visit http://0install.net.
9 from zeroinstall import _
10 import os, platform, re, subprocess, sys
11 from logging import warn, info
12 from zeroinstall.injector import namespaces, model, arch
13 from zeroinstall.support import basedir
15 _dotted_ints = '[0-9]+(?:\.[0-9]+)*'
17 # This matches a version number that would be a valid Zero Install version without modification
18 _zeroinstall_regexp = '(?:%s)(?:-(?:pre|rc|post|)(?:%s))*' % (_dotted_ints, _dotted_ints)
20 # This matches the interesting bits of distribution version numbers
21 _version_regexp = '(%s)(-r%s)?' % (_zeroinstall_regexp, _dotted_ints)
23 # We try to do updates atomically without locking, but we don't worry too much about
24 # duplicate entries or being a little out of sync with the on-disk copy.
25 class Cache(object):
26 def __init__(self, cache_leaf, source, format):
27 """Maintain a cache file (e.g. ~/.cache/0install.net/injector/$name).
28 If the size or mtime of $source has changed, or the cache
29 format version if different, reset the cache first."""
30 self.cache_leaf = cache_leaf
31 self.source = source
32 self.format = format
33 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
34 namespaces.config_prog)
35 self.cached_for = {} # Attributes of source when cache was created
36 try:
37 self._load_cache()
38 except Exception as ex:
39 info(_("Failed to load cache (%s). Flushing..."), ex)
40 self.flush()
42 def flush(self):
43 # Wipe the cache
44 try:
45 info = os.stat(self.source)
46 mtime = int(info.st_mtime)
47 size = info.st_size
48 except Exception as ex:
49 warn("Failed to stat %s: %s", self.source, ex)
50 mtime = size = 0
51 self.cache = {}
52 import tempfile
53 tmp, tmp_name = tempfile.mkstemp(dir = self.cache_dir)
54 data = "mtime=%d\nsize=%d\nformat=%d\n\n" % (mtime, size, self.format)
55 while data:
56 wrote = os.write(tmp, data)
57 data = data[wrote:]
58 os.rename(tmp_name, os.path.join(self.cache_dir, self.cache_leaf))
60 self._load_cache()
62 # Populate self.cache from our saved cache file.
63 # Throws an exception if the cache doesn't exist or has the wrong format.
64 def _load_cache(self):
65 self.cache = cache = {}
66 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
67 try:
68 for line in stream:
69 line = line.strip()
70 if not line:
71 break
72 key, value = line.split('=', 1)
73 if key in ('mtime', 'size', 'format'):
74 self.cached_for[key] = int(value)
76 self._check_valid()
78 for line in stream:
79 key, value = line.split('=', 1)
80 cache[key] = value[:-1]
81 finally:
82 stream.close()
84 # Check the source file hasn't changed since we created the cache
85 def _check_valid(self):
86 info = os.stat(self.source)
87 if self.cached_for['mtime'] != int(info.st_mtime):
88 raise Exception("Modification time of %s has changed" % self.source)
89 if self.cached_for['size'] != info.st_size:
90 raise Exception("Size of %s has changed" % self.source)
91 if self.cached_for.get('format', None) != self.format:
92 raise Exception("Format of cache has changed")
94 def get(self, key):
95 try:
96 self._check_valid()
97 except Exception as ex:
98 info(_("Cache needs to be refreshed: %s"), ex)
99 self.flush()
100 return None
101 else:
102 return self.cache.get(key, None)
104 def put(self, key, value):
105 cache_path = os.path.join(self.cache_dir, self.cache_leaf)
106 self.cache[key] = value
107 try:
108 stream = file(cache_path, 'a')
109 try:
110 stream.write('%s=%s\n' % (key, value))
111 finally:
112 stream.close()
113 except Exception as ex:
114 warn("Failed to write to cache %s: %s=%s: %s", cache_path, key, value, ex)
116 def try_cleanup_distro_version(version):
117 """Try to turn a distribution version string into one readable by Zero Install.
118 We do this by stripping off anything we can't parse.
119 @return: the part we understood, or None if we couldn't parse anything
120 @rtype: str"""
121 if ':' in version:
122 version = version.split(':')[1] # Skip 'epoch'
123 version = version.replace('_', '-')
124 match = re.match(_version_regexp, version)
125 if match:
126 version, revision = match.groups()
127 if revision is None:
128 return version
129 else:
130 return '%s-%s' % (version, revision[2:])
131 return None
133 class Distribution(object):
134 """Represents a distribution with which we can integrate.
135 Sub-classes should specialise this to integrate with the package managers of
136 particular distributions. This base class ignores the native package manager.
137 @since: 0.28
139 _packagekit = None
141 def get_package_info(self, package, factory):
142 """Get information about the given package.
143 Add zero or more implementations using the factory (typically at most two
144 will be added; the currently installed version and the latest available).
145 @param package: package name (e.g. "gimp")
146 @type package: str
147 @param factory: function for creating new DistributionImplementation objects from IDs
148 @type factory: str -> L{model.DistributionImplementation}
150 return
152 def get_score(self, distribution):
153 """Indicate how closely the host distribution matches this one.
154 The <package-implementation> with the highest score is passed
155 to L{Distribution.get_package_info}. If several elements get
156 the same score, get_package_info is called for all of them.
157 @param distribution: a distribution name
158 @type distribution: str
159 @return: an integer, or None if there is no match at all
160 @rtype: int | None
162 return 0
164 def get_feed(self, master_feed):
165 """Generate a feed containing information about distribution packages.
166 This should immediately return a feed containing an implementation for the
167 package if it's already installed. Information about versions that could be
168 installed using the distribution's package manager can be added asynchronously
169 later (see L{fetch_candidates}).
170 @param master_feed: feed containing the <package-implementation> elements
171 @type master_feed: L{model.ZeroInstallFeed}
172 @rtype: L{model.ZeroInstallFeed}"""
174 feed = model.ZeroInstallFeed(None)
175 feed.url = 'distribution:' + master_feed.url
177 for item, item_attrs in master_feed.get_package_impls(self):
178 package = item_attrs.get('package', None)
179 if package is None:
180 raise model.InvalidInterface(_("Missing 'package' attribute on %s") % item)
182 def factory(id, only_if_missing = False, installed = True):
183 assert id.startswith('package:')
184 if id in feed.implementations:
185 if only_if_missing:
186 return None
187 warn(_("Duplicate ID '%s' for DistributionImplementation"), id)
188 impl = model.DistributionImplementation(feed, id, self)
189 feed.implementations[id] = impl
191 impl.installed = installed
192 impl.metadata = item_attrs
194 item_main = item_attrs.get('main', None)
195 if item_main and not item_main.startswith('/'):
196 raise model.InvalidInterface(_("'main' attribute must be absolute, but '%s' doesn't start with '/'!") %
197 item_main)
198 impl.main = item_main
199 impl.upstream_stability = model.packaged
201 return impl
203 self.get_package_info(package, factory)
205 if master_feed.url == 'http://repo.roscidus.com/python/python' and all(not impl.installed for impl in feed.implementations.values()):
206 # Hack: we can support Python on platforms with unsupported package managers
207 # by adding the implementation of Python running us now to the list.
208 python_version = '.'.join([str(v) for v in sys.version_info if isinstance(v, int)])
209 impl_id = 'package:host:python:' + python_version
210 assert impl_id not in feed.implementations
211 impl = model.DistributionImplementation(feed, impl_id, self)
212 impl.installed = True
213 impl.version = model.parse_version(python_version)
214 impl.main = sys.executable
215 impl.upstream_stability = model.packaged
216 impl.machine = host_machine # (hopefully)
217 feed.implementations[impl_id] = impl
219 return feed
221 def fetch_candidates(self, master_feed):
222 """Collect information about versions we could install using
223 the distribution's package manager. On success, the distribution
224 feed in iface_cache is updated.
225 @return: a L{tasks.Blocker} if the task is in progress, or None if not"""
226 if self.packagekit.available:
227 package_names = [item.getAttribute("package") for item, item_attrs in master_feed.get_package_impls(self)]
228 return self.packagekit.fetch_candidates(package_names)
230 @property
231 def packagekit(self):
232 """For use by subclasses.
233 @rtype: L{packagekit.PackageKit}"""
234 if not self._packagekit:
235 from zeroinstall.injector import packagekit
236 self._packagekit = packagekit.PackageKit()
237 return self._packagekit
239 class CachedDistribution(Distribution):
240 """For distributions where querying the package database is slow (e.g. requires running
241 an external command), we cache the results.
242 @since: 0.39
243 @deprecated: use Cache instead
246 def __init__(self, db_status_file):
247 """@param db_status_file: update the cache when the timestamp of this file changes"""
248 self._status_details = os.stat(db_status_file)
250 self.versions = {}
251 self.cache_dir = basedir.save_cache_path(namespaces.config_site,
252 namespaces.config_prog)
254 try:
255 self._load_cache()
256 except Exception as ex:
257 info(_("Failed to load distribution database cache (%s). Regenerating..."), ex)
258 try:
259 self.generate_cache()
260 self._load_cache()
261 except Exception as ex:
262 warn(_("Failed to regenerate distribution database cache: %s"), ex)
264 def _load_cache(self):
265 """Load {cache_leaf} cache file into self.versions if it is available and up-to-date.
266 Throws an exception if the cache should be (re)created."""
267 stream = file(os.path.join(self.cache_dir, self.cache_leaf))
269 cache_version = None
270 for line in stream:
271 if line == '\n':
272 break
273 name, value = line.split(': ')
274 if name == 'mtime' and int(value) != int(self._status_details.st_mtime):
275 raise Exception(_("Modification time of package database file has changed"))
276 if name == 'size' and int(value) != self._status_details.st_size:
277 raise Exception(_("Size of package database file has changed"))
278 if name == 'version':
279 cache_version = int(value)
280 else:
281 raise Exception(_('Invalid cache format (bad header)'))
283 if cache_version is None:
284 raise Exception(_('Old cache format'))
286 versions = self.versions
287 for line in stream:
288 package, version, zi_arch = line[:-1].split('\t')
289 versionarch = (version, intern(zi_arch))
290 if package not in versions:
291 versions[package] = [versionarch]
292 else:
293 versions[package].append(versionarch)
295 def _write_cache(self, cache):
296 #cache.sort() # Might be useful later; currently we don't care
297 import tempfile
298 fd, tmpname = tempfile.mkstemp(prefix = 'zeroinstall-cache-tmp',
299 dir = self.cache_dir)
300 try:
301 stream = os.fdopen(fd, 'wb')
302 stream.write('version: 2\n')
303 stream.write('mtime: %d\n' % int(self._status_details.st_mtime))
304 stream.write('size: %d\n' % self._status_details.st_size)
305 stream.write('\n')
306 for line in cache:
307 stream.write(line + '\n')
308 stream.close()
310 os.rename(tmpname,
311 os.path.join(self.cache_dir,
312 self.cache_leaf))
313 except:
314 os.unlink(tmpname)
315 raise
317 # Maps machine type names used in packages to their Zero Install versions
318 _canonical_machine = {
319 'all' : '*',
320 'any' : '*',
321 'noarch' : '*',
322 '(none)' : '*',
323 'amd64': 'x86_64',
324 'i386': 'i386',
325 'i486': 'i486',
326 'i586': 'i586',
327 'i686': 'i686',
328 'ppc64': 'ppc64',
329 'ppc': 'ppc',
332 host_machine = arch.canonicalize_machine(platform.uname()[4])
333 def canonical_machine(package_machine):
334 machine = _canonical_machine.get(package_machine, None)
335 if machine is None:
336 # Safe default if we can't understand the arch
337 return host_machine
338 return machine
340 class DebianDistribution(Distribution):
341 """A dpkg-based distribution."""
343 cache_leaf = 'dpkg-status.cache'
345 def __init__(self, dpkg_status, pkgcache):
346 self.dpkg_cache = Cache('dpkg-status.cache', dpkg_status, 2)
347 self.apt_cache = {}
349 def _query_installed_package(self, package):
350 null = os.open('/dev/null', os.O_WRONLY)
351 child = subprocess.Popen(["dpkg-query", "-W", "--showformat=${Version}\t${Architecture}\t${Status}\n", "--", package],
352 stdout = subprocess.PIPE, stderr = null)
353 os.close(null)
354 stdout, stderr = child.communicate()
355 child.wait()
356 for line in stdout.split('\n'):
357 if not line: continue
358 version, debarch, status = line.split('\t', 2)
359 if not status.endswith(' installed'): continue
360 clean_version = try_cleanup_distro_version(version)
361 if clean_version:
362 return '%s\t%s' % (clean_version, canonical_machine(debarch.strip()))
363 else:
364 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
366 return '-'
368 def get_package_info(self, package, factory):
369 # Add any already-installed package...
370 installed_cached_info = self._get_dpkg_info(package)
372 if installed_cached_info != '-':
373 installed_version, machine = installed_cached_info.split('\t')
374 impl = factory('package:deb:%s:%s:%s' % (package, installed_version, machine))
375 impl.version = model.parse_version(installed_version)
376 if machine != '*':
377 impl.machine = machine
378 else:
379 installed_version = None
381 # Add any uninstalled candidates (note: only one of these two methods will add anything)
383 # From PackageKit...
384 self.packagekit.get_candidates(package, factory, 'package:deb')
386 # From apt-cache...
387 cached = self.apt_cache.get(package, None)
388 if cached:
389 candidate_version = cached['version']
390 candidate_arch = cached['arch']
391 if candidate_version and candidate_version != installed_version:
392 impl = factory('package:deb:%s:%s:%s' % (package, candidate_version, candidate_arch), installed = False)
393 impl.version = model.parse_version(candidate_version)
394 if candidate_arch != '*':
395 impl.machine = candidate_arch
396 def install(handler):
397 raise model.SafeException(_("This program depends on '%s', which is a package that is available through your distribution. "
398 "Please install it manually using your distribution's tools and try again.") % package)
399 impl.download_sources.append(model.DistributionSource(package, cached['size'], install, needs_confirmation = False))
401 def get_score(self, disto_name):
402 return int(disto_name == 'Debian')
404 def _get_dpkg_info(self, package):
405 installed_cached_info = self.dpkg_cache.get(package)
406 if installed_cached_info == None:
407 installed_cached_info = self._query_installed_package(package)
408 self.dpkg_cache.put(package, installed_cached_info)
410 return installed_cached_info
412 def fetch_candidates(self, master_feed):
413 package_names = [item.getAttribute("package") for item, item_attrs in master_feed.get_package_impls(self)]
415 if self.packagekit.available:
416 return self.packagekit.fetch_candidates(package_names)
418 # No PackageKit. Use apt-cache directly.
419 for package in package_names:
420 # Check to see whether we could get a newer version using apt-get
421 try:
422 null = os.open('/dev/null', os.O_WRONLY)
423 child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null)
424 os.close(null)
426 arch = version = size = None
427 for line in child.stdout:
428 line = line.strip()
429 if line.startswith('Version: '):
430 version = line[9:]
431 version = try_cleanup_distro_version(version)
432 elif line.startswith('Architecture: '):
433 arch = canonical_machine(line[14:].strip())
434 elif line.startswith('Size: '):
435 size = int(line[6:].strip())
436 if version and arch:
437 cached = {'version': version, 'arch': arch, 'size': size}
438 else:
439 cached = None
440 child.wait()
441 except Exception as ex:
442 warn("'apt-cache show %s' failed: %s", package, ex)
443 cached = None
444 # (multi-arch support? can there be multiple candidates?)
445 self.apt_cache[package] = cached
447 class RPMDistribution(CachedDistribution):
448 """An RPM-based distribution."""
450 cache_leaf = 'rpm-status.cache'
452 def generate_cache(self):
453 cache = []
455 for line in os.popen("rpm -qa --qf='%{NAME}\t%{VERSION}-%{RELEASE}\t%{ARCH}\n'"):
456 package, version, rpmarch = line.split('\t', 2)
457 if package == 'gpg-pubkey':
458 continue
459 zi_arch = canonical_machine(rpmarch.strip())
460 clean_version = try_cleanup_distro_version(version)
461 if clean_version:
462 cache.append('%s\t%s\t%s' % (package, clean_version, zi_arch))
463 else:
464 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': package})
466 self._write_cache(cache)
468 def get_package_info(self, package, factory):
469 # Add installed versions...
470 versions = self.versions.get(package, [])
472 for version, machine in versions:
473 impl = factory('package:rpm:%s:%s:%s' % (package, version, machine))
474 impl.version = model.parse_version(version)
475 if machine != '*':
476 impl.machine = machine
478 # Add any uninstalled candidates found by PackageKit
479 self.packagekit.get_candidates(package, factory, 'package:rpm')
481 def get_score(self, disto_name):
482 return int(disto_name == 'RPM')
484 class SlackDistribution(Distribution):
485 """A Slack-based distribution."""
487 def __init__(self, packages_dir):
488 self._packages_dir = packages_dir
490 def get_package_info(self, package, factory):
491 # Add installed versions...
492 for entry in os.listdir(self._packages_dir):
493 name, version, arch, build = entry.rsplit('-', 3)
494 if name == package:
495 zi_arch = canonical_machine(arch)
496 clean_version = try_cleanup_distro_version("%s-%s" % (version, build))
497 if not clean_version:
498 warn(_("Can't parse distribution version '%(version)s' for package '%(package)s'"), {'version': version, 'package': name})
499 continue
501 impl = factory('package:slack:%s:%s:%s' % \
502 (package, clean_version, zi_arch))
503 impl.version = model.parse_version(clean_version)
504 if zi_arch != '*':
505 impl.machine = zi_arch
507 # Add any uninstalled candidates found by PackageKit
508 self.packagekit.get_candidates(package, factory, 'package:slack')
510 def get_score(self, disto_name):
511 return int(disto_name == 'Slack')
513 class GentooDistribution(Distribution):
515 def __init__(self, pkgdir):
516 self._pkgdir = pkgdir
518 def get_package_info(self, package, factory):
519 # Add installed versions...
520 _version_start_reqexp = '-[0-9]'
522 if package.count('/') != 1: return
524 category, leafname = package.split('/')
525 category_dir = os.path.join(self._pkgdir, category)
526 match_prefix = leafname + '-'
528 if not os.path.isdir(category_dir): return
530 for filename in os.listdir(category_dir):
531 if filename.startswith(match_prefix) and filename[len(match_prefix)].isdigit():
532 name = file(os.path.join(category_dir, filename, 'PF')).readline().strip()
534 match = re.search(_version_start_reqexp, name)
535 if match is None:
536 warn(_('Cannot parse version from Gentoo package named "%(name)s"'), {'name': name})
537 continue
538 else:
539 version = try_cleanup_distro_version(name[match.start() + 1:])
541 if category == 'app-emulation' and name.startswith('emul-'):
542 __, __, machine, __ = name.split('-', 3)
543 else:
544 machine, __ = file(os.path.join(category_dir, filename, 'CHOST')).readline().split('-', 1)
545 machine = arch.canonicalize_machine(machine)
547 impl = factory('package:gentoo:%s:%s:%s' % \
548 (package, version, machine))
549 impl.version = model.parse_version(version)
550 impl.machine = machine
552 # Add any uninstalled candidates found by PackageKit
553 self.packagekit.get_candidates(package, factory, 'package:gentoo')
555 def get_score(self, disto_name):
556 return int(disto_name == 'Gentoo')
558 class PortsDistribution(Distribution):
560 def __init__(self, pkgdir):
561 self._pkgdir = pkgdir
563 def get_package_info(self, package, factory):
564 _name_version_regexp = '^(.+)-([^-]+)$'
566 nameversion = re.compile(_name_version_regexp)
567 for pkgname in os.listdir(self._pkgdir):
568 pkgdir = os.path.join(self._pkgdir, pkgname)
569 if not os.path.isdir(pkgdir): continue
571 #contents = file(os.path.join(pkgdir, '+CONTENTS')).readline().strip()
573 match = nameversion.search(pkgname)
574 if match is None:
575 warn(_('Cannot parse version from Ports package named "%(pkgname)s"'), {'pkgname': pkgname})
576 continue
577 else:
578 name = match.group(1)
579 if name != package:
580 continue
581 version = try_cleanup_distro_version(match.group(2))
583 machine = host_machine
585 impl = factory('package:ports:%s:%s:%s' % \
586 (package, version, machine))
587 impl.version = model.parse_version(version)
588 impl.machine = machine
590 def get_score(self, disto_name):
591 return int(disto_name == 'Ports')
593 _host_distribution = None
594 def get_host_distribution():
595 """Get a Distribution suitable for the host operating system.
596 Calling this twice will return the same object.
597 @rtype: L{Distribution}"""
598 global _host_distribution
599 if not _host_distribution:
600 dpkg_db_status = '/var/lib/dpkg/status'
601 pkgcache = '/var/cache/apt/pkgcache.bin'
602 _rpm_db = '/var/lib/rpm/Packages'
603 _slack_db = '/var/log/packages'
604 _pkg_db = '/var/db/pkg'
606 if os.path.isdir(_pkg_db):
607 if sys.platform.startswith("linux"):
608 _host_distribution = GentooDistribution(_pkg_db)
609 elif sys.platform.startswith("freebsd"):
610 _host_distribution = PortsDistribution(_pkg_db)
611 elif os.access(dpkg_db_status, os.R_OK):
612 _host_distribution = DebianDistribution(dpkg_db_status, pkgcache)
613 elif os.path.isfile(_rpm_db):
614 _host_distribution = RPMDistribution(_rpm_db)
615 elif os.path.isdir(_slack_db):
616 _host_distribution = SlackDistribution(_slack_db)
617 else:
618 _host_distribution = Distribution()
620 return _host_distribution