Lib/plat-mac/pimp.py

   1 """Package Install Manager for Python.
   2
   3 This is currently a MacOSX-only strawman implementation.
   4 Despite other rumours the name stands for "Packman IMPlementation".
   5
   6 Tools to allow easy installation of packages. The idea is that there is
   7 an online XML database per (platform, python-version) containing packages
   8 known to work with that combination. This module contains tools for getting
   9 and parsing the database, testing whether packages are installed, computing
  10 dependencies and installing packages.
  11
  12 There is a minimal main program that works as a command line tool, but the
  13 intention is that the end user will use this through a GUI.
  14 """
  15
  16 from warnings import warnpy3k
  17 warnpy3k("In 3.x, the pimp module is removed.", stacklevel=2)
  18
  19 import sys
  20 import os
  21 import subprocess
  22 import urllib
  23 import urllib2
  24 import urlparse
  25 import plistlib
  26 import distutils.util
  27 import distutils.sysconfig
  28 import hashlib
  29 import tarfile
  30 import tempfile
  31 import shutil
  32 import time
  33
  34 __all__ = ["PimpPreferences", "PimpDatabase", "PimpPackage", "main",
  35     "getDefaultDatabase", "PIMP_VERSION", "main"]
  36
  37 _scriptExc_NotInstalled = "pimp._scriptExc_NotInstalled"
  38 _scriptExc_OldInstalled = "pimp._scriptExc_OldInstalled"
  39 _scriptExc_BadInstalled = "pimp._scriptExc_BadInstalled"
  40
  41 NO_EXECUTE=0
  42
  43 PIMP_VERSION="0.5"
  44
  45 # Flavors:
  46 # source: setup-based package
  47 # binary: tar (or other) archive created with setup.py bdist.
  48 # installer: something that can be opened
  49 DEFAULT_FLAVORORDER=['source', 'binary', 'installer']
  50 DEFAULT_DOWNLOADDIR='/tmp'
  51 DEFAULT_BUILDDIR='/tmp'
  52 DEFAULT_INSTALLDIR=distutils.sysconfig.get_python_lib()
  53 DEFAULT_PIMPDATABASE_FMT="http://www.python.org/packman/version-%s/%s-%s-%s-%s-%s.plist"
  54
  55 def getDefaultDatabase(experimental=False):
  56     if experimental:
  57         status = "exp"
  58     else:
  59         status = "prod"
  60
  61     major, minor, micro, state, extra = sys.version_info
  62     pyvers = '%d.%d' % (major, minor)
  63     if micro == 0 and state != 'final':
  64         pyvers = pyvers + '%s%d' % (state, extra)
  65
  66     longplatform = distutils.util.get_platform()
  67     osname, release, machine = longplatform.split('-')
  68     # For some platforms we may want to differentiate between
  69     # installation types
  70     if osname == 'darwin':
  71         if sys.prefix.startswith('/System/Library/Frameworks/Python.framework'):
  72             osname = 'darwin_apple'
  73         elif sys.prefix.startswith('/Library/Frameworks/Python.framework'):
  74             osname = 'darwin_macpython'
  75         # Otherwise we don't know...
  76     # Now we try various URLs by playing with the release string.
  77     # We remove numbers off the end until we find a match.
  78     rel = release
  79     while True:
  80         url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, rel, machine)
  81         try:
  82             urllib2.urlopen(url)
  83         except urllib2.HTTPError, arg:
  84             pass
  85         else:
  86             break
  87         if not rel:
  88             # We're out of version numbers to try. Use the
  89             # full release number, this will give a reasonable
  90             # error message later
  91             url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, release, machine)
  92             break
  93         idx = rel.rfind('.')
  94         if idx < 0:
  95             rel = ''
  96         else:
  97             rel = rel[:idx]
  98     return url
  99
 100 def _cmd(output, dir, *cmditems):
 101     """Internal routine to run a shell command in a given directory."""
 102
 103     cmd = ("cd \"%s\"; " % dir) + " ".join(cmditems)
 104     if output:
 105         output.write("+ %s\n" % cmd)
 106     if NO_EXECUTE:
 107         return 0
 108     child = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
 109                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 110     child.stdin.close()
 111     while 1:
 112         line = child.stdout.readline()
 113         if not line:
 114             break
 115         if output:
 116             output.write(line)
 117     return child.wait()
 118
 119 class PimpDownloader:
 120     """Abstract base class - Downloader for archives"""
 121
 122     def __init__(self, argument,
 123             dir="",
 124             watcher=None):
 125         self.argument = argument
 126         self._dir = dir
 127         self._watcher = watcher
 128
 129     def download(self, url, filename, output=None):
 130         return None
 131
 132     def update(self, str):
 133         if self._watcher:
 134             return self._watcher.update(str)
 135         return True
 136
 137 class PimpCurlDownloader(PimpDownloader):
 138
 139     def download(self, url, filename, output=None):
 140         self.update("Downloading %s..." % url)
 141         exitstatus = _cmd(output, self._dir,
 142                     "curl",
 143                     "--output", filename,
 144                     url)
 145         self.update("Downloading %s: finished" % url)
 146         return (not exitstatus)
 147
 148 class PimpUrllibDownloader(PimpDownloader):
 149
 150     def download(self, url, filename, output=None):
 151         output = open(filename, 'wb')
 152         self.update("Downloading %s: opening connection" % url)
 153         keepgoing = True
 154         download = urllib2.urlopen(url)
 155         if download.headers.has_key("content-length"):
 156             length = long(download.headers['content-length'])
 157         else:
 158             length = -1
 159
 160         data = download.read(4096) #read 4K at a time
 161         dlsize = 0
 162         lasttime = 0
 163         while keepgoing:
 164             dlsize = dlsize + len(data)
 165             if len(data) == 0:
 166                 #this is our exit condition
 167                 break
 168             output.write(data)
 169             if int(time.time()) != lasttime:
 170                 # Update at most once per second
 171                 lasttime = int(time.time())
 172                 if length == -1:
 173                     keepgoing = self.update("Downloading %s: %d bytes..." % (url, dlsize))
 174                 else:
 175                     keepgoing = self.update("Downloading %s: %d%% (%d bytes)..." % (url, int(100.0*dlsize/length), dlsize))
 176             data = download.read(4096)
 177         if keepgoing:
 178             self.update("Downloading %s: finished" % url)
 179         return keepgoing
 180
 181 class PimpUnpacker:
 182     """Abstract base class - Unpacker for archives"""
 183
 184     _can_rename = False
 185
 186     def __init__(self, argument,
 187             dir="",
 188             renames=[],
 189             watcher=None):
 190         self.argument = argument
 191         if renames and not self._can_rename:
 192             raise RuntimeError, "This unpacker cannot rename files"
 193         self._dir = dir
 194         self._renames = renames
 195         self._watcher = watcher
 196
 197     def unpack(self, archive, output=None, package=None):
 198         return None
 199
 200     def update(self, str):
 201         if self._watcher:
 202             return self._watcher.update(str)
 203         return True
 204
 205 class PimpCommandUnpacker(PimpUnpacker):
 206     """Unpack archives by calling a Unix utility"""
 207
 208     _can_rename = False
 209
 210     def unpack(self, archive, output=None, package=None):
 211         cmd = self.argument % archive
 212         if _cmd(output, self._dir, cmd):
 213             return "unpack command failed"
 214
 215 class PimpTarUnpacker(PimpUnpacker):
 216     """Unpack tarfiles using the builtin tarfile module"""
 217
 218     _can_rename = True
 219
 220     def unpack(self, archive, output=None, package=None):
 221         tf = tarfile.open(archive, "r")
 222         members = tf.getmembers()
 223         skip = []
 224         if self._renames:
 225             for member in members:
 226                 for oldprefix, newprefix in self._renames:
 227                     if oldprefix[:len(self._dir)] == self._dir:
 228                         oldprefix2 = oldprefix[len(self._dir):]
 229                     else:
 230                         oldprefix2 = None
 231                     if member.name[:len(oldprefix)] == oldprefix:
 232                         if newprefix is None:
 233                             skip.append(member)
 234                             #print 'SKIP', member.name
 235                         else:
 236                             member.name = newprefix + member.name[len(oldprefix):]
 237                             print '    ', member.name
 238                         break
 239                     elif oldprefix2 and member.name[:len(oldprefix2)] == oldprefix2:
 240                         if newprefix is None:
 241                             skip.append(member)
 242                             #print 'SKIP', member.name
 243                         else:
 244                             member.name = newprefix + member.name[len(oldprefix2):]
 245                             #print '    ', member.name
 246                         break
 247                 else:
 248                     skip.append(member)
 249                     #print '????', member.name
 250         for member in members:
 251             if member in skip:
 252                 self.update("Skipping %s" % member.name)
 253                 continue
 254             self.update("Extracting %s" % member.name)
 255             tf.extract(member, self._dir)
 256         if skip:
 257             names = [member.name for member in skip if member.name[-1] != '/']
 258             if package:
 259                 names = package.filterExpectedSkips(names)
 260             if names:
 261                 return "Not all files were unpacked: %s" % " ".join(names)
 262
 263 ARCHIVE_FORMATS = [
 264     (".tar.Z", PimpTarUnpacker, None),
 265     (".taz", PimpTarUnpacker, None),
 266     (".tar.gz", PimpTarUnpacker, None),
 267     (".tgz", PimpTarUnpacker, None),
 268     (".tar.bz", PimpTarUnpacker, None),
 269     (".zip", PimpCommandUnpacker, "unzip \"%s\""),
 270 ]
 271
 272 class PimpPreferences:
 273     """Container for per-user preferences, such as the database to use
 274     and where to install packages."""
 275
 276     def __init__(self,
 277             flavorOrder=None,
 278             downloadDir=None,
 279             buildDir=None,
 280             installDir=None,
 281             pimpDatabase=None):
 282         if not flavorOrder:
 283             flavorOrder = DEFAULT_FLAVORORDER
 284         if not downloadDir:
 285             downloadDir = DEFAULT_DOWNLOADDIR
 286         if not buildDir:
 287             buildDir = DEFAULT_BUILDDIR
 288         if not pimpDatabase:
 289             pimpDatabase = getDefaultDatabase()
 290         self.setInstallDir(installDir)
 291         self.flavorOrder = flavorOrder
 292         self.downloadDir = downloadDir
 293         self.buildDir = buildDir
 294         self.pimpDatabase = pimpDatabase
 295         self.watcher = None
 296
 297     def setWatcher(self, watcher):
 298         self.watcher = watcher
 299
 300     def setInstallDir(self, installDir=None):
 301         if installDir:
 302             # Installing to non-standard location.
 303             self.installLocations = [
 304                 ('--install-lib', installDir),
 305                 ('--install-headers', None),
 306                 ('--install-scripts', None),
 307                 ('--install-data', None)]
 308         else:
 309             installDir = DEFAULT_INSTALLDIR
 310             self.installLocations = []
 311         self.installDir = installDir
 312
 313     def isUserInstall(self):
 314         return self.installDir != DEFAULT_INSTALLDIR
 315
 316     def check(self):
 317         """Check that the preferences make sense: directories exist and are
 318         writable, the install directory is on sys.path, etc."""
 319
 320         rv = ""
 321         RWX_OK = os.R_OK|os.W_OK|os.X_OK
 322         if not os.path.exists(self.downloadDir):
 323             rv += "Warning: Download directory \"%s\" does not exist\n" % self.downloadDir
 324         elif not os.access(self.downloadDir, RWX_OK):
 325             rv += "Warning: Download directory \"%s\" is not writable or not readable\n" % self.downloadDir
 326         if not os.path.exists(self.buildDir):
 327             rv += "Warning: Build directory \"%s\" does not exist\n" % self.buildDir
 328         elif not os.access(self.buildDir, RWX_OK):
 329             rv += "Warning: Build directory \"%s\" is not writable or not readable\n" % self.buildDir
 330         if not os.path.exists(self.installDir):
 331             rv += "Warning: Install directory \"%s\" does not exist\n" % self.installDir
 332         elif not os.access(self.installDir, RWX_OK):
 333             rv += "Warning: Install directory \"%s\" is not writable or not readable\n" % self.installDir
 334         else:
 335             installDir = os.path.realpath(self.installDir)
 336             for p in sys.path:
 337                 try:
 338                     realpath = os.path.realpath(p)
 339                 except:
 340                     pass
 341                 if installDir == realpath:
 342                     break
 343             else:
 344                 rv += "Warning: Install directory \"%s\" is not on sys.path\n" % self.installDir
 345         return rv
 346
 347     def compareFlavors(self, left, right):
 348         """Compare two flavor strings. This is part of your preferences
 349         because whether the user prefers installing from source or binary is."""
 350         if left in self.flavorOrder:
 351             if right in self.flavorOrder:
 352                 return cmp(self.flavorOrder.index(left), self.flavorOrder.index(right))
 353             return -1
 354         if right in self.flavorOrder:
 355             return 1
 356         return cmp(left, right)
 357
 358 class PimpDatabase:
 359     """Class representing a pimp database. It can actually contain
 360     information from multiple databases through inclusion, but the
 361     toplevel database is considered the master, as its maintainer is
 362     "responsible" for the contents."""
 363
 364     def __init__(self, prefs):
 365         self._packages = []
 366         self.preferences = prefs
 367         self._url = ""
 368         self._urllist = []
 369         self._version = ""
 370         self._maintainer = ""
 371         self._description = ""
 372
 373     # Accessor functions
 374     def url(self): return self._url
 375     def version(self): return self._version
 376     def maintainer(self): return self._maintainer
 377     def description(self): return self._description
 378
 379     def close(self):
 380         """Clean up"""
 381         self._packages = []
 382         self.preferences = None
 383
 384     def appendURL(self, url, included=0):
 385         """Append packages from the database with the given URL.
 386         Only the first database should specify included=0, so the
 387         global information (maintainer, description) get stored."""
 388
 389         if url in self._urllist:
 390             return
 391         self._urllist.append(url)
 392         fp = urllib2.urlopen(url).fp
 393         plistdata = plistlib.Plist.fromFile(fp)
 394         # Test here for Pimp version, etc
 395         if included:
 396             version = plistdata.get('Version')
 397             if version and version > self._version:
 398                 sys.stderr.write("Warning: included database %s is for pimp version %s\n" %
 399                     (url, version))
 400         else:
 401             self._version = plistdata.get('Version')
 402             if not self._version:
 403                 sys.stderr.write("Warning: database has no Version information\n")
 404             elif self._version > PIMP_VERSION:
 405                 sys.stderr.write("Warning: database version %s newer than pimp version %s\n"
 406                     % (self._version, PIMP_VERSION))
 407             self._maintainer = plistdata.get('Maintainer', '')
 408             self._description = plistdata.get('Description', '').strip()
 409             self._url = url
 410         self._appendPackages(plistdata['Packages'], url)
 411         others = plistdata.get('Include', [])
 412         for o in others:
 413             o = urllib.basejoin(url, o)
 414             self.appendURL(o, included=1)
 415
 416     def _appendPackages(self, packages, url):
 417         """Given a list of dictionaries containing package
 418         descriptions create the PimpPackage objects and append them
 419         to our internal storage."""
 420
 421         for p in packages:
 422             p = dict(p)
 423             if p.has_key('Download-URL'):
 424                 p['Download-URL'] = urllib.basejoin(url, p['Download-URL'])
 425             flavor = p.get('Flavor')
 426             if flavor == 'source':
 427                 pkg = PimpPackage_source(self, p)
 428             elif flavor == 'binary':
 429                 pkg = PimpPackage_binary(self, p)
 430             elif flavor == 'installer':
 431                 pkg = PimpPackage_installer(self, p)
 432             elif flavor == 'hidden':
 433                 pkg = PimpPackage_installer(self, p)
 434             else:
 435                 pkg = PimpPackage(self, dict(p))
 436             self._packages.append(pkg)
 437
 438     def list(self):
 439         """Return a list of all PimpPackage objects in the database."""
 440
 441         return self._packages
 442
 443     def listnames(self):
 444         """Return a list of names of all packages in the database."""
 445
 446         rv = []
 447         for pkg in self._packages:
 448             rv.append(pkg.fullname())
 449         rv.sort()
 450         return rv
 451
 452     def dump(self, pathOrFile):
 453         """Dump the contents of the database to an XML .plist file.
 454
 455         The file can be passed as either a file object or a pathname.
 456         All data, including included databases, is dumped."""
 457
 458         packages = []
 459         for pkg in self._packages:
 460             packages.append(pkg.dump())
 461         plistdata = {
 462             'Version': self._version,
 463             'Maintainer': self._maintainer,
 464             'Description': self._description,
 465             'Packages': packages
 466             }
 467         plist = plistlib.Plist(**plistdata)
 468         plist.write(pathOrFile)
 469
 470     def find(self, ident):
 471         """Find a package. The package can be specified by name
 472         or as a dictionary with name, version and flavor entries.
 473
 474         Only name is obligatory. If there are multiple matches the
 475         best one (higher version number, flavors ordered according to
 476         users' preference) is returned."""
 477
 478         if type(ident) == str:
 479             # Remove ( and ) for pseudo-packages
 480             if ident[0] == '(' and ident[-1] == ')':
 481                 ident = ident[1:-1]
 482             # Split into name-version-flavor
 483             fields = ident.split('-')
 484             if len(fields) < 1 or len(fields) > 3:
 485                 return None
 486             name = fields[0]
 487             if len(fields) > 1:
 488                 version = fields[1]
 489             else:
 490                 version = None
 491             if len(fields) > 2:
 492                 flavor = fields[2]
 493             else:
 494                 flavor = None
 495         else:
 496             name = ident['Name']
 497             version = ident.get('Version')
 498             flavor = ident.get('Flavor')
 499         found = None
 500         for p in self._packages:
 501             if name == p.name() and \
 502                     (not version or version == p.version()) and \
 503                     (not flavor or flavor == p.flavor()):
 504                 if not found or found < p:
 505                     found = p
 506         return found
 507
 508 ALLOWED_KEYS = [
 509     "Name",
 510     "Version",
 511     "Flavor",
 512     "Description",
 513     "Home-page",
 514     "Download-URL",
 515     "Install-test",
 516     "Install-command",
 517     "Pre-install-command",
 518     "Post-install-command",
 519     "Prerequisites",
 520     "MD5Sum",
 521     "User-install-skips",
 522     "Systemwide-only",
 523 ]
 524
 525 class PimpPackage:
 526     """Class representing a single package."""
 527
 528     def __init__(self, db, plistdata):
 529         self._db = db
 530         name = plistdata["Name"]
 531         for k in plistdata.keys():
 532             if not k in ALLOWED_KEYS:
 533                 sys.stderr.write("Warning: %s: unknown key %s\n" % (name, k))
 534         self._dict = plistdata
 535
 536     def __getitem__(self, key):
 537         return self._dict[key]
 538
 539     def name(self): return self._dict['Name']
 540     def version(self): return self._dict.get('Version')
 541     def flavor(self): return self._dict.get('Flavor')
 542     def description(self): return self._dict['Description'].strip()
 543     def shortdescription(self): return self.description().splitlines()[0]
 544     def homepage(self): return self._dict.get('Home-page')
 545     def downloadURL(self): return self._dict.get('Download-URL')
 546     def systemwideOnly(self): return self._dict.get('Systemwide-only')
 547
 548     def fullname(self):
 549         """Return the full name "name-version-flavor" of a package.
 550
 551         If the package is a pseudo-package, something that cannot be
 552         installed through pimp, return the name in (parentheses)."""
 553
 554         rv = self._dict['Name']
 555         if self._dict.has_key('Version'):
 556             rv = rv + '-%s' % self._dict['Version']
 557         if self._dict.has_key('Flavor'):
 558             rv = rv + '-%s' % self._dict['Flavor']
 559         if self._dict.get('Flavor') == 'hidden':
 560             # Pseudo-package, show in parentheses
 561             rv = '(%s)' % rv
 562         return rv
 563
 564     def dump(self):
 565         """Return a dict object containing the information on the package."""
 566         return self._dict
 567
 568     def __cmp__(self, other):
 569         """Compare two packages, where the "better" package sorts lower."""
 570
 571         if not isinstance(other, PimpPackage):
 572             return cmp(id(self), id(other))
 573         if self.name() != other.name():
 574             return cmp(self.name(), other.name())
 575         if self.version() != other.version():
 576             return -cmp(self.version(), other.version())
 577         return self._db.preferences.compareFlavors(self.flavor(), other.flavor())
 578
 579     def installed(self):
 580         """Test wheter the package is installed.
 581
 582         Returns two values: a status indicator which is one of
 583         "yes", "no", "old" (an older version is installed) or "bad"
 584         (something went wrong during the install test) and a human
 585         readable string which may contain more details."""
 586
 587         namespace = {
 588             "NotInstalled": _scriptExc_NotInstalled,
 589             "OldInstalled": _scriptExc_OldInstalled,
 590             "BadInstalled": _scriptExc_BadInstalled,
 591             "os": os,
 592             "sys": sys,
 593             }
 594         installTest = self._dict['Install-test'].strip() + '\n'
 595         try:
 596             exec installTest in namespace
 597         except ImportError, arg:
 598             return "no", str(arg)
 599         except _scriptExc_NotInstalled, arg:
 600             return "no", str(arg)
 601         except _scriptExc_OldInstalled, arg:
 602             return "old", str(arg)
 603         except _scriptExc_BadInstalled, arg:
 604             return "bad", str(arg)
 605         except:
 606             sys.stderr.write("-------------------------------------\n")
 607             sys.stderr.write("---- %s: install test got exception\n" % self.fullname())
 608             sys.stderr.write("---- source:\n")
 609             sys.stderr.write(installTest)
 610             sys.stderr.write("---- exception:\n")
 611             import traceback
 612             traceback.print_exc(file=sys.stderr)
 613             if self._db._maintainer:
 614                 sys.stderr.write("---- Please copy this and mail to %s\n" % self._db._maintainer)
 615             sys.stderr.write("-------------------------------------\n")
 616             return "bad", "Package install test got exception"
 617         return "yes", ""
 618
 619     def prerequisites(self):
 620         """Return a list of prerequisites for this package.
 621
 622         The list contains 2-tuples, of which the first item is either
 623         a PimpPackage object or None, and the second is a descriptive
 624         string. The first item can be None if this package depends on
 625         something that isn't pimp-installable, in which case the descriptive
 626         string should tell the user what to do."""
 627
 628         rv = []
 629         if not self._dict.get('Download-URL'):
 630             # For pseudo-packages that are already installed we don't
 631             # return an error message
 632             status, _  = self.installed()
 633             if status == "yes":
 634                 return []
 635             return [(None,
 636                 "Package %s cannot be installed automatically, see the description" %
 637                     self.fullname())]
 638         if self.systemwideOnly() and self._db.preferences.isUserInstall():
 639             return [(None,
 640                 "Package %s can only be installed system-wide" %
 641                     self.fullname())]
 642         if not self._dict.get('Prerequisites'):
 643             return []
 644         for item in self._dict['Prerequisites']:
 645             if type(item) == str:
 646                 pkg = None
 647                 descr = str(item)
 648             else:
 649                 name = item['Name']
 650                 if item.has_key('Version'):
 651                     name = name + '-' + item['Version']
 652                 if item.has_key('Flavor'):
 653                     name = name + '-' + item['Flavor']
 654                 pkg = self._db.find(name)
 655                 if not pkg:
 656                     descr = "Requires unknown %s"%name
 657                 else:
 658                     descr = pkg.shortdescription()
 659             rv.append((pkg, descr))
 660         return rv
 661
 662
 663     def downloadPackageOnly(self, output=None):
 664         """Download a single package, if needed.
 665
 666         An MD5 signature is used to determine whether download is needed,
 667         and to test that we actually downloaded what we expected.
 668         If output is given it is a file-like object that will receive a log
 669         of what happens.
 670
 671         If anything unforeseen happened the method returns an error message
 672         string.
 673         """
 674
 675         scheme, loc, path, query, frag = urlparse.urlsplit(self._dict['Download-URL'])
 676         path = urllib.url2pathname(path)
 677         filename = os.path.split(path)[1]
 678         self.archiveFilename = os.path.join(self._db.preferences.downloadDir, filename)
 679         if not self._archiveOK():
 680             if scheme == 'manual':
 681                 return "Please download package manually and save as %s" % self.archiveFilename
 682             downloader = PimpUrllibDownloader(None, self._db.preferences.downloadDir,
 683                 watcher=self._db.preferences.watcher)
 684             if not downloader.download(self._dict['Download-URL'],
 685                     self.archiveFilename, output):
 686                 return "download command failed"
 687         if not os.path.exists(self.archiveFilename) and not NO_EXECUTE:
 688             return "archive not found after download"
 689         if not self._archiveOK():
 690             return "archive does not have correct MD5 checksum"
 691
 692     def _archiveOK(self):
 693         """Test an archive. It should exist and the MD5 checksum should be correct."""
 694
 695         if not os.path.exists(self.archiveFilename):
 696             return 0
 697         if not self._dict.get('MD5Sum'):
 698             sys.stderr.write("Warning: no MD5Sum for %s\n" % self.fullname())
 699             return 1
 700         data = open(self.archiveFilename, 'rb').read()
 701         checksum = hashlib.md5(data).hexdigest()
 702         return checksum == self._dict['MD5Sum']
 703
 704     def unpackPackageOnly(self, output=None):
 705         """Unpack a downloaded package archive."""
 706
 707         filename = os.path.split(self.archiveFilename)[1]
 708         for ext, unpackerClass, arg in ARCHIVE_FORMATS:
 709             if filename[-len(ext):] == ext:
 710                 break
 711         else:
 712             return "unknown extension for archive file: %s" % filename
 713         self.basename = filename[:-len(ext)]
 714         unpacker = unpackerClass(arg, dir=self._db.preferences.buildDir,
 715                 watcher=self._db.preferences.watcher)
 716         rv = unpacker.unpack(self.archiveFilename, output=output)
 717         if rv:
 718             return rv
 719
 720     def installPackageOnly(self, output=None):
 721         """Default install method, to be overridden by subclasses"""
 722         return "%s: This package needs to be installed manually (no support for flavor=\"%s\")" \
 723             % (self.fullname(), self._dict.get(flavor, ""))
 724
 725     def installSinglePackage(self, output=None):
 726         """Download, unpack and install a single package.
 727
 728         If output is given it should be a file-like object and it
 729         will receive a log of what happened."""
 730
 731         if not self._dict.get('Download-URL'):
 732             return "%s: This package needs to be installed manually (no Download-URL field)" % self.fullname()
 733         msg = self.downloadPackageOnly(output)
 734         if msg:
 735             return "%s: download: %s" % (self.fullname(), msg)
 736
 737         msg = self.unpackPackageOnly(output)
 738         if msg:
 739             return "%s: unpack: %s" % (self.fullname(), msg)
 740
 741         return self.installPackageOnly(output)
 742
 743     def beforeInstall(self):
 744         """Bookkeeping before installation: remember what we have in site-packages"""
 745         self._old_contents = os.listdir(self._db.preferences.installDir)
 746
 747     def afterInstall(self):
 748         """Bookkeeping after installation: interpret any new .pth files that have
 749         appeared"""
 750
 751         new_contents = os.listdir(self._db.preferences.installDir)
 752         for fn in new_contents:
 753             if fn in self._old_contents:
 754                 continue
 755             if fn[-4:] != '.pth':
 756                 continue
 757             fullname = os.path.join(self._db.preferences.installDir, fn)
 758             f = open(fullname)
 759             for line in f.readlines():
 760                 if not line:
 761                     continue
 762                 if line[0] == '#':
 763                     continue
 764                 if line[:6] == 'import':
 765                     exec line
 766                     continue
 767                 if line[-1] == '\n':
 768                     line = line[:-1]
 769                 if not os.path.isabs(line):
 770                     line = os.path.join(self._db.preferences.installDir, line)
 771                 line = os.path.realpath(line)
 772                 if not line in sys.path:
 773                     sys.path.append(line)
 774
 775     def filterExpectedSkips(self, names):
 776         """Return a list that contains only unpexpected skips"""
 777         if not self._db.preferences.isUserInstall():
 778             return names
 779         expected_skips = self._dict.get('User-install-skips')
 780         if not expected_skips:
 781             return names
 782         newnames = []
 783         for name in names:
 784             for skip in expected_skips:
 785                 if name[:len(skip)] == skip:
 786                     break
 787             else:
 788                 newnames.append(name)
 789         return newnames
 790
 791 class PimpPackage_binary(PimpPackage):
 792
 793     def unpackPackageOnly(self, output=None):
 794         """We don't unpack binary packages until installing"""
 795         pass
 796
 797     def installPackageOnly(self, output=None):
 798         """Install a single source package.
 799
 800         If output is given it should be a file-like object and it
 801         will receive a log of what happened."""
 802
 803         if self._dict.has_key('Install-command'):
 804             return "%s: Binary package cannot have Install-command" % self.fullname()
 805
 806         if self._dict.has_key('Pre-install-command'):
 807             if _cmd(output, '/tmp', self._dict['Pre-install-command']):
 808                 return "pre-install %s: running \"%s\" failed" % \
 809                     (self.fullname(), self._dict['Pre-install-command'])
 810
 811         self.beforeInstall()
 812
 813         # Install by unpacking
 814         filename = os.path.split(self.archiveFilename)[1]
 815         for ext, unpackerClass, arg in ARCHIVE_FORMATS:
 816             if filename[-len(ext):] == ext:
 817                 break
 818         else:
 819             return "%s: unknown extension for archive file: %s" % (self.fullname(), filename)
 820         self.basename = filename[:-len(ext)]
 821
 822         install_renames = []
 823         for k, newloc in self._db.preferences.installLocations:
 824             if not newloc:
 825                 continue
 826             if k == "--install-lib":
 827                 oldloc = DEFAULT_INSTALLDIR
 828             else:
 829                 return "%s: Don't know installLocation %s" % (self.fullname(), k)
 830             install_renames.append((oldloc, newloc))
 831
 832         unpacker = unpackerClass(arg, dir="/", renames=install_renames)
 833         rv = unpacker.unpack(self.archiveFilename, output=output, package=self)
 834         if rv:
 835             return rv
 836
 837         self.afterInstall()
 838
 839         if self._dict.has_key('Post-install-command'):
 840             if _cmd(output, '/tmp', self._dict['Post-install-command']):
 841                 return "%s: post-install: running \"%s\" failed" % \
 842                     (self.fullname(), self._dict['Post-install-command'])
 843
 844         return None
 845
 846
 847 class PimpPackage_source(PimpPackage):
 848
 849     def unpackPackageOnly(self, output=None):
 850         """Unpack a source package and check that setup.py exists"""
 851         PimpPackage.unpackPackageOnly(self, output)
 852         # Test that a setup script has been create
 853         self._buildDirname = os.path.join(self._db.preferences.buildDir, self.basename)
 854         setupname = os.path.join(self._buildDirname, "setup.py")
 855         if not os.path.exists(setupname) and not NO_EXECUTE:
 856             return "no setup.py found after unpack of archive"
 857
 858     def installPackageOnly(self, output=None):
 859         """Install a single source package.
 860
 861         If output is given it should be a file-like object and it
 862         will receive a log of what happened."""
 863
 864         if self._dict.has_key('Pre-install-command'):
 865             if _cmd(output, self._buildDirname, self._dict['Pre-install-command']):
 866                 return "pre-install %s: running \"%s\" failed" % \
 867                     (self.fullname(), self._dict['Pre-install-command'])
 868
 869         self.beforeInstall()
 870         installcmd = self._dict.get('Install-command')
 871         if installcmd and self._install_renames:
 872             return "Package has install-command and can only be installed to standard location"
 873         # This is the "bit-bucket" for installations: everything we don't
 874         # want. After installation we check that it is actually empty
 875         unwanted_install_dir = None
 876         if not installcmd:
 877             extra_args = ""
 878             for k, v in self._db.preferences.installLocations:
 879                 if not v:
 880                     # We don't want these files installed. Send them
 881                     # to the bit-bucket.
 882                     if not unwanted_install_dir:
 883                         unwanted_install_dir = tempfile.mkdtemp()
 884                     v = unwanted_install_dir
 885                 extra_args = extra_args + " %s \"%s\"" % (k, v)
 886             installcmd = '"%s" setup.py install %s' % (sys.executable, extra_args)
 887         if _cmd(output, self._buildDirname, installcmd):
 888             return "install %s: running \"%s\" failed" % \
 889                 (self.fullname(), installcmd)
 890         if unwanted_install_dir and os.path.exists(unwanted_install_dir):
 891             unwanted_files = os.listdir(unwanted_install_dir)
 892             if unwanted_files:
 893                 rv = "Warning: some files were not installed: %s" % " ".join(unwanted_files)
 894             else:
 895                 rv = None
 896             shutil.rmtree(unwanted_install_dir)
 897             return rv
 898
 899         self.afterInstall()
 900
 901         if self._dict.has_key('Post-install-command'):
 902             if _cmd(output, self._buildDirname, self._dict['Post-install-command']):
 903                 return "post-install %s: running \"%s\" failed" % \
 904                     (self.fullname(), self._dict['Post-install-command'])
 905         return None
 906
 907 class PimpPackage_installer(PimpPackage):
 908
 909     def unpackPackageOnly(self, output=None):
 910         """We don't unpack dmg packages until installing"""
 911         pass
 912
 913     def installPackageOnly(self, output=None):
 914         """Install a single source package.
 915
 916         If output is given it should be a file-like object and it
 917         will receive a log of what happened."""
 918
 919         if self._dict.has_key('Post-install-command'):
 920             return "%s: Installer package cannot have Post-install-command" % self.fullname()
 921
 922         if self._dict.has_key('Pre-install-command'):
 923             if _cmd(output, '/tmp', self._dict['Pre-install-command']):
 924                 return "pre-install %s: running \"%s\" failed" % \
 925                     (self.fullname(), self._dict['Pre-install-command'])
 926
 927         self.beforeInstall()
 928
 929         installcmd = self._dict.get('Install-command')
 930         if installcmd:
 931             if '%' in installcmd:
 932                 installcmd = installcmd % self.archiveFilename
 933         else:
 934             installcmd = 'open \"%s\"' % self.archiveFilename
 935         if _cmd(output, "/tmp", installcmd):
 936             return '%s: install command failed (use verbose for details)' % self.fullname()
 937         return '%s: downloaded and opened. Install manually and restart Package Manager' % self.archiveFilename
 938
 939 class PimpInstaller:
 940     """Installer engine: computes dependencies and installs
 941     packages in the right order."""
 942
 943     def __init__(self, db):
 944         self._todo = []
 945         self._db = db
 946         self._curtodo = []
 947         self._curmessages = []
 948
 949     def __contains__(self, package):
 950         return package in self._todo
 951
 952     def _addPackages(self, packages):
 953         for package in packages:
 954             if not package in self._todo:
 955                 self._todo.append(package)
 956
 957     def _prepareInstall(self, package, force=0, recursive=1):
 958         """Internal routine, recursive engine for prepareInstall.
 959
 960         Test whether the package is installed and (if not installed
 961         or if force==1) prepend it to the temporary todo list and
 962         call ourselves recursively on all prerequisites."""
 963
 964         if not force:
 965             status, message = package.installed()
 966             if status == "yes":
 967                 return
 968         if package in self._todo or package in self._curtodo:
 969             return
 970         self._curtodo.insert(0, package)
 971         if not recursive:
 972             return
 973         prereqs = package.prerequisites()
 974         for pkg, descr in prereqs:
 975             if pkg:
 976                 self._prepareInstall(pkg, False, recursive)
 977             else:
 978                 self._curmessages.append("Problem with dependency: %s" % descr)
 979
 980     def prepareInstall(self, package, force=0, recursive=1):
 981         """Prepare installation of a package.
 982
 983         If the package is already installed and force is false nothing
 984         is done. If recursive is true prerequisites are installed first.
 985
 986         Returns a list of packages (to be passed to install) and a list
 987         of messages of any problems encountered.
 988         """
 989
 990         self._curtodo = []
 991         self._curmessages = []
 992         self._prepareInstall(package, force, recursive)
 993         rv = self._curtodo, self._curmessages
 994         self._curtodo = []
 995         self._curmessages = []
 996         return rv
 997
 998     def install(self, packages, output):
 999         """Install a list of packages."""
1000
1001         self._addPackages(packages)
1002         status = []
1003         for pkg in self._todo:
1004             msg = pkg.installSinglePackage(output)
1005             if msg:
1006                 status.append(msg)
1007         return status
1008
1009
1010
1011 def _run(mode, verbose, force, args, prefargs, watcher):
1012     """Engine for the main program"""
1013
1014     prefs = PimpPreferences(**prefargs)
1015     if watcher:
1016         prefs.setWatcher(watcher)
1017     rv = prefs.check()
1018     if rv:
1019         sys.stdout.write(rv)
1020     db = PimpDatabase(prefs)
1021     db.appendURL(prefs.pimpDatabase)
1022
1023     if mode == 'dump':
1024         db.dump(sys.stdout)
1025     elif mode =='list':
1026         if not args:
1027             args = db.listnames()
1028         print "%-20.20s\t%s" % ("Package", "Description")
1029         print
1030         for pkgname in args:
1031             pkg = db.find(pkgname)
1032             if pkg:
1033                 description = pkg.shortdescription()
1034                 pkgname = pkg.fullname()
1035             else:
1036                 description = 'Error: no such package'
1037             print "%-20.20s\t%s" % (pkgname, description)
1038             if verbose:
1039                 print "\tHome page:\t", pkg.homepage()
1040                 try:
1041                     print "\tDownload URL:\t", pkg.downloadURL()
1042                 except KeyError:
1043                     pass
1044                 description = pkg.description()
1045                 description = '\n\t\t\t\t\t'.join(description.splitlines())
1046                 print "\tDescription:\t%s" % description
1047     elif mode =='status':
1048         if not args:
1049             args = db.listnames()
1050             print "%-20.20s\t%s\t%s" % ("Package", "Installed", "Message")
1051             print
1052         for pkgname in args:
1053             pkg = db.find(pkgname)
1054             if pkg:
1055                 status, msg = pkg.installed()
1056                 pkgname = pkg.fullname()
1057             else:
1058                 status = 'error'
1059                 msg = 'No such package'
1060             print "%-20.20s\t%-9.9s\t%s" % (pkgname, status, msg)
1061             if verbose and status == "no":
1062                 prereq = pkg.prerequisites()
1063                 for pkg, msg in prereq:
1064                     if not pkg:
1065                         pkg = ''
1066                     else:
1067                         pkg = pkg.fullname()
1068                     print "%-20.20s\tRequirement: %s %s" % ("", pkg, msg)
1069     elif mode == 'install':
1070         if not args:
1071             print 'Please specify packages to install'
1072             sys.exit(1)
1073         inst = PimpInstaller(db)
1074         for pkgname in args:
1075             pkg = db.find(pkgname)
1076             if not pkg:
1077                 print '%s: No such package' % pkgname
1078                 continue
1079             list, messages = inst.prepareInstall(pkg, force)
1080             if messages and not force:
1081                 print "%s: Not installed:" % pkgname
1082                 for m in messages:
1083                     print "\t", m
1084             else:
1085                 if verbose:
1086                     output = sys.stdout
1087                 else:
1088                     output = None
1089                 messages = inst.install(list, output)
1090                 if messages:
1091                     print "%s: Not installed:" % pkgname
1092                     for m in messages:
1093                         print "\t", m
1094
1095 def main():
1096     """Minimal commandline tool to drive pimp."""
1097
1098     import getopt
1099     def _help():
1100         print "Usage: pimp [options] -s [package ...]  List installed status"
1101         print "       pimp [options] -l [package ...]  Show package information"
1102         print "       pimp [options] -i package ...    Install packages"
1103         print "       pimp -d                          Dump database to stdout"
1104         print "       pimp -V                          Print version number"
1105         print "Options:"
1106         print "       -v     Verbose"
1107         print "       -f     Force installation"
1108         print "       -D dir Set destination directory"
1109         print "              (default: %s)" % DEFAULT_INSTALLDIR
1110         print "       -u url URL for database"
1111         sys.exit(1)
1112
1113     class _Watcher:
1114         def update(self, msg):
1115             sys.stderr.write(msg + '\r')
1116             return 1
1117
1118     try:
1119         opts, args = getopt.getopt(sys.argv[1:], "slifvdD:Vu:")
1120     except getopt.GetoptError:
1121         _help()
1122     if not opts and not args:
1123         _help()
1124     mode = None
1125     force = 0
1126     verbose = 0
1127     prefargs = {}
1128     watcher = None
1129     for o, a in opts:
1130         if o == '-s':
1131             if mode:
1132                 _help()
1133             mode = 'status'
1134         if o == '-l':
1135             if mode:
1136                 _help()
1137             mode = 'list'
1138         if o == '-d':
1139             if mode:
1140                 _help()
1141             mode = 'dump'
1142         if o == '-V':
1143             if mode:
1144                 _help()
1145             mode = 'version'
1146         if o == '-i':
1147             mode = 'install'
1148         if o == '-f':
1149             force = 1
1150         if o == '-v':
1151             verbose = 1
1152             watcher = _Watcher()
1153         if o == '-D':
1154             prefargs['installDir'] = a
1155         if o == '-u':
1156             prefargs['pimpDatabase'] = a
1157     if not mode:
1158         _help()
1159     if mode == 'version':
1160         print 'Pimp version %s; module name is %s' % (PIMP_VERSION, __name__)
1161     else:
1162         _run(mode, verbose, force, args, prefargs, watcher)
1163
1164 # Finally, try to update ourselves to a newer version.
1165 # If the end-user updates pimp through pimp the new version
1166 # will be called pimp_update and live in site-packages
1167 # or somewhere similar
1168 if __name__ != 'pimp_update':
1169     try:
1170         import pimp_update
1171     except ImportError:
1172         pass
1173     else:
1174         if pimp_update.PIMP_VERSION <= PIMP_VERSION:
1175             import warnings
1176             warnings.warn("pimp_update is version %s, not newer than pimp version %s" %
1177                 (pimp_update.PIMP_VERSION, PIMP_VERSION))
1178         else:
1179             from pimp_update import *
1180
1181 if __name__ == '__main__':
1182     main()