utils/external_packages.py

   1 #!/usr/bin/python
   2 #
   3 # Please keep this code python 2.4 compatible and stand alone.
   4
   5 import logging, os, shutil, sys, tempfile, time, urllib2
   6 import subprocess, re
   7 from autotest_lib.client.common_lib import utils
   8
   9 _READ_SIZE = 64*1024
  10 _MAX_PACKAGE_SIZE = 100*1024*1024
  11
  12
  13 class Error(Exception):
  14     """Local exception to be raised by code in this file."""
  15
  16 class FetchError(Error):
  17     """Failed to fetch a package from any of its listed URLs."""
  18
  19
  20 def _checksum_file(full_path):
  21     """@returns The hex checksum of a file given its pathname."""
  22     inputfile = open(full_path, 'rb')
  23     try:
  24         hex_sum = utils.hash('sha1', inputfile.read()).hexdigest()
  25     finally:
  26         inputfile.close()
  27     return hex_sum
  28
  29
  30 def system(commandline):
  31     """Same as os.system(commandline) but logs the command first."""
  32     logging.info(commandline)
  33     return os.system(commandline)
  34
  35
  36 def find_top_of_autotest_tree():
  37     """@returns The full path to the top of the autotest directory tree."""
  38     dirname = os.path.dirname(__file__)
  39     autotest_dir = os.path.abspath(os.path.join(dirname, '..'))
  40     return autotest_dir
  41
  42
  43 class ExternalPackage(object):
  44     """
  45     Defines an external package with URLs to fetch its sources from and
  46     a build_and_install() method to unpack it, build it and install it
  47     beneath our own autotest/site-packages directory.
  48
  49     Base Class.  Subclass this to define packages.
  50
  51     Attributes:
  52       @attribute urls - A tuple of URLs to try fetching the package from.
  53       @attribute local_filename - A local filename to use when saving the
  54               fetched package.
  55       @attribute hex_sum - The hex digest (currently SHA1) of this package
  56               to be used to verify its contents.
  57       @attribute module_name - The installed python module name to be used for
  58               for a version check.  Defaults to the lower case class name with
  59               the word Package stripped off.
  60       @attribute version - The desired minimum package version.
  61       @attribute os_requirements - A dictionary mapping a file pathname on the
  62               the OS distribution to a likely name of a package the user
  63               needs to install on their system in order to get this file.
  64       @attribute name - Read only, the printable name of the package.
  65       @attribute subclasses - This class attribute holds a list of all defined
  66               subclasses.  It is constructed dynamically using the metaclass.
  67     """
  68     subclasses = []
  69     urls = ()
  70     local_filename = None
  71     hex_sum = None
  72     module_name = None
  73     version = None
  74     os_requirements = None
  75
  76
  77     class __metaclass__(type):
  78         """Any time a subclass is defined, add it to our list."""
  79         def __init__(mcs, name, bases, dict):
  80             if name != 'ExternalPackage':
  81                 mcs.subclasses.append(mcs)
  82
  83
  84     def __init__(self):
  85         self.verified_package = ''
  86         if not self.module_name:
  87             self.module_name = self.name.lower()
  88         self.installed_version = ''
  89
  90
  91     @property
  92     def name(self):
  93         """Return the class name with any trailing 'Package' stripped off."""
  94         class_name = self.__class__.__name__
  95         if class_name.endswith('Package'):
  96             return class_name[:-len('Package')]
  97         return class_name
  98
  99
 100     def is_needed(self, unused_install_dir):
 101         """@returns True if self.module_name needs to be built and installed."""
 102         if not self.module_name or not self.version:
 103             logging.warning('version and module_name required for '
 104                             'is_needed() check to work.')
 105             return True
 106         try:
 107             module = __import__(self.module_name)
 108         except ImportError, e:
 109             logging.info("%s isn't present. Will install.", self.module_name)
 110             return True
 111         self.installed_version = self._get_installed_version_from_module(module)
 112         logging.info('imported %s version %s.', self.module_name,
 113                      self.installed_version)
 114         return self.version > self.installed_version
 115
 116
 117     def _get_installed_version_from_module(self, module):
 118         """Ask our module its version string and return it or '' if unknown."""
 119         try:
 120             return module.__version__
 121         except AttributeError:
 122             logging.error('could not get version from %s', module)
 123             return ''
 124
 125
 126     def _build_and_install(self, install_dir):
 127         """Subclasses MUST provide their own implementation."""
 128         raise NotImplementedError
 129
 130
 131     def _build_and_install_current_dir(self, install_dir):
 132         """
 133         Subclasses that use _build_and_install_from_package() MUST provide
 134         their own implementation of this method.
 135         """
 136         raise NotImplementedError
 137
 138
 139     def build_and_install(self, install_dir):
 140         """
 141         Builds and installs the package.  It must have been fetched already.
 142
 143         @param install_dir - The package installation directory.  If it does
 144             not exist it will be created.
 145         """
 146         if not self.verified_package:
 147             raise Error('Must call fetch() first.  - %s' % self.name)
 148         self._check_os_requirements()
 149         return self._build_and_install(install_dir)
 150
 151
 152     def _check_os_requirements(self):
 153         if not self.os_requirements:
 154             return
 155         failed = False
 156         for file_name, package_name in self.os_requirements.iteritems():
 157             if not os.path.exists(file_name):
 158                 failed = True
 159                 logging.error('File %s not found, %s needs it.',
 160                               file_name, self.name)
 161                 logging.error('Perhaps you need to install something similar '
 162                               'to the %s package for OS first.', package_name)
 163         if failed:
 164             raise Error('Missing OS requirements for %s.  (see above)' %
 165                         self.name)
 166
 167
 168     def _build_and_install_current_dir_setup_py(self, install_dir):
 169         """For use as a _build_and_install_current_dir implementation."""
 170         egg_path = self._build_egg_using_setup_py(setup_py='setup.py')
 171         if not egg_path:
 172             return False
 173         return self._install_from_egg(install_dir, egg_path)
 174
 175
 176     def _build_and_install_current_dir_setupegg_py(self, install_dir):
 177         """For use as a _build_and_install_current_dir implementation."""
 178         egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py')
 179         if not egg_path:
 180             return False
 181         return self._install_from_egg(install_dir, egg_path)
 182
 183
 184     def _build_and_install_current_dir_noegg(self, install_dir):
 185         if not self._build_using_setup_py():
 186             return False
 187         return self._install_using_setup_py_and_rsync(install_dir)
 188
 189
 190     def _build_and_install_from_package(self, install_dir):
 191         """
 192         This method may be used as a _build_and_install() implementation
 193         for subclasses if they implement _build_and_install_current_dir().
 194
 195         Extracts the .tar.gz file, chdirs into the extracted directory
 196         (which is assumed to match the tar filename) and calls
 197         _build_and_isntall_current_dir from there.
 198
 199         Afterwards the build (regardless of failure) extracted .tar.gz
 200         directory is cleaned up.
 201
 202         @returns True on success, False otherwise.
 203
 204         @raises OSError If the expected extraction directory does not exist.
 205         """
 206         self._extract_compressed_package()
 207         if self.verified_package.endswith('.tar.gz'):
 208             extension = '.tar.gz'
 209         elif self.verified_package.endswith('.tar.bz2'):
 210             extension = '.tar.bz2'
 211         elif self.verified_package.endswith('.zip'):
 212             extension = '.zip'
 213         else:
 214             raise Error('Unexpected package file extension on %s' %
 215                         self.verified_package)
 216         os.chdir(os.path.dirname(self.verified_package))
 217         os.chdir(self.local_filename[:-len(extension)])
 218         extracted_dir = os.getcwd()
 219         try:
 220             return self._build_and_install_current_dir(install_dir)
 221         finally:
 222             os.chdir(os.path.join(extracted_dir, '..'))
 223             shutil.rmtree(extracted_dir)
 224
 225
 226     def _extract_compressed_package(self):
 227         """Extract the fetched compressed .tar or .zip within its directory."""
 228         if not self.verified_package:
 229             raise Error('Package must have been fetched first.')
 230         os.chdir(os.path.dirname(self.verified_package))
 231         if self.verified_package.endswith('gz'):
 232             status = system("tar -xzf '%s'" % self.verified_package)
 233         elif self.verified_package.endswith('bz2'):
 234             status = system("tar -xjf '%s'" % self.verified_package)
 235         elif self.verified_package.endswith('zip'):
 236             status = system("unzip '%s'" % self.verified_package)
 237         else:
 238             raise Error('Unknown compression suffix on %s.' %
 239                         self.verified_package)
 240         if status:
 241             raise Error('tar failed with %s' % (status,))
 242
 243
 244     def _build_using_setup_py(self, setup_py='setup.py'):
 245         """
 246         Assuming the cwd is the extracted python package, execute a simple
 247         python setup.py build.
 248
 249         @param setup_py - The name of the setup.py file to execute.
 250
 251         @returns True on success, False otherwise.
 252         """
 253         if not os.path.exists(setup_py):
 254             raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
 255         status = system("'%s' %s build" % (sys.executable, setup_py))
 256         if status:
 257             logging.error('%s build failed.' % self.name)
 258             return False
 259         return True
 260
 261
 262     def _build_egg_using_setup_py(self, setup_py='setup.py'):
 263         """
 264         Assuming the cwd is the extracted python package, execute a simple
 265         python setup.py bdist_egg.
 266
 267         @param setup_py - The name of the setup.py file to execute.
 268
 269         @returns The relative path to the resulting egg file or '' on failure.
 270         """
 271         if not os.path.exists(setup_py):
 272             raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
 273         egg_subdir = 'dist'
 274         if os.path.isdir(egg_subdir):
 275             shutil.rmtree(egg_subdir)
 276         status = system("'%s' %s bdist_egg" % (sys.executable, setup_py))
 277         if status:
 278             logging.error('bdist_egg of setuptools failed.')
 279             return ''
 280         # I've never seen a bdist_egg lay multiple .egg files.
 281         for filename in os.listdir(egg_subdir):
 282             if filename.endswith('.egg'):
 283                 return os.path.join(egg_subdir, filename)
 284
 285
 286     def _install_from_egg(self, install_dir, egg_path):
 287         """
 288         Install a module from an egg file by unzipping the necessary parts
 289         into install_dir.
 290
 291         @param install_dir - The installation directory.
 292         @param egg_path - The pathname of the egg file.
 293         """
 294         status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path))
 295         if status:
 296             logging.error('unzip of %s failed', egg_path)
 297             return False
 298         egg_info = os.path.join(install_dir, 'EGG-INFO')
 299         if os.path.isdir(egg_info):
 300             shutil.rmtree(egg_info)
 301         return True
 302
 303
 304     def _get_temp_dir(self):
 305         return tempfile.mkdtemp(dir='/var/tmp')
 306
 307
 308     def _site_packages_path(self, temp_dir):
 309         # This makes assumptions about what python setup.py install
 310         # does when given a prefix.  Is this always correct?
 311         python_xy = 'python%s' % sys.version[:3]
 312         return os.path.join(temp_dir, 'lib', python_xy, 'site-packages')
 313
 314
 315     def _install_using_setup_py_and_rsync(self, install_dir,
 316                                           setup_py='setup.py',
 317                                           temp_dir=None):
 318         """
 319         Assuming the cwd is the extracted python package, execute a simple:
 320
 321           python setup.py install --prefix=BLA
 322
 323         BLA will be a temporary directory that everything installed will
 324         be picked out of and rsynced to the appropriate place under
 325         install_dir afterwards.
 326
 327         Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/
 328         directory tree that setuptools created and moves all installed
 329         site-packages directly up into install_dir itself.
 330
 331         @param install_dir the directory for the install to happen under.
 332         @param setup_py - The name of the setup.py file to execute.
 333
 334         @returns True on success, False otherwise.
 335         """
 336         if not os.path.exists(setup_py):
 337             raise Error('%s does not exist in %s' % (setup_py, os.getcwd()))
 338
 339         if temp_dir is None:
 340             temp_dir = self._get_temp_dir()
 341
 342         try:
 343             status = system("'%s' %s install --no-compile --prefix='%s'"
 344                             % (sys.executable, setup_py, temp_dir))
 345             if status:
 346                 logging.error('%s install failed.' % self.name)
 347                 return False
 348
 349             if os.path.isdir(os.path.join(temp_dir, 'lib')):
 350                 # NOTE: This ignores anything outside of the lib/ dir that
 351                 # was installed.
 352                 temp_site_dir = self._site_packages_path(temp_dir)
 353             else:
 354                 temp_site_dir = temp_dir
 355
 356             status = system("rsync -r '%s/' '%s/'" %
 357                             (temp_site_dir, install_dir))
 358             if status:
 359                 logging.error('%s rsync to install_dir failed.' % self.name)
 360                 return False
 361             return True
 362         finally:
 363             shutil.rmtree(temp_dir)
 364
 365
 366
 367     def _build_using_make(self, install_dir):
 368         """Build the current package using configure/make.
 369
 370         @returns True on success, False otherwise.
 371         """
 372         install_prefix = os.path.join(install_dir, 'usr', 'local')
 373         status = system('./configure --prefix=%s' % install_prefix)
 374         if status:
 375             logging.error('./configure failed for %s', self.name)
 376             return False
 377         status = system('make')
 378         if status:
 379             logging.error('make failed for %s', self.name)
 380             return False
 381         status = system('make check')
 382         if status:
 383             logging.error('make check failed for %s', self.name)
 384             return False
 385         return True
 386
 387
 388     def _install_using_make(self):
 389         """Install the current package using make install.
 390
 391         Assumes the install path was set up while running ./configure (in
 392         _build_using_make()).
 393
 394         @returns True on success, False otherwise.
 395         """
 396         status = system('make install')
 397         return status == 0
 398
 399
 400     def fetch(self, dest_dir):
 401         """
 402         Fetch the package from one its URLs and save it in dest_dir.
 403
 404         If the the package already exists in dest_dir and the checksum
 405         matches this code will not fetch it again.
 406
 407         Sets the 'verified_package' attribute with the destination pathname.
 408
 409         @param dest_dir - The destination directory to save the local file.
 410             If it does not exist it will be created.
 411
 412         @returns A boolean indicating if we the package is now in dest_dir.
 413         @raises FetchError - When something unexpected happens.
 414         """
 415         if not os.path.exists(dest_dir):
 416             os.makedirs(dest_dir)
 417         local_path = os.path.join(dest_dir, self.local_filename)
 418
 419         # If the package exists, verify its checksum and be happy if it is good.
 420         if os.path.exists(local_path):
 421             actual_hex_sum = _checksum_file(local_path)
 422             if self.hex_sum == actual_hex_sum:
 423                 logging.info('Good checksum for existing %s package.',
 424                              self.name)
 425                 self.verified_package = local_path
 426                 return True
 427             logging.warning('Bad checksum for existing %s package.  '
 428                             'Re-downloading', self.name)
 429             os.rename(local_path, local_path + '.wrong-checksum')
 430
 431         # Download the package from one of its urls, rejecting any if the
 432         # checksum does not match.
 433         for url in self.urls:
 434             logging.info('Fetching %s', url)
 435             try:
 436                 url_file = urllib2.urlopen(url)
 437             except (urllib2.URLError, EnvironmentError):
 438                 logging.warning('Could not fetch %s package from %s.',
 439                                 self.name, url)
 440                 continue
 441             data_length = int(url_file.info().get('Content-Length',
 442                                                   _MAX_PACKAGE_SIZE))
 443             if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE:
 444                 raise FetchError('%s from %s fails Content-Length %d '
 445                                  'sanity check.' % (self.name, url,
 446                                                     data_length))
 447             checksum = utils.hash('sha1')
 448             total_read = 0
 449             output = open(local_path, 'wb')
 450             try:
 451                 while total_read < data_length:
 452                     data = url_file.read(_READ_SIZE)
 453                     if not data:
 454                         break
 455                     output.write(data)
 456                     checksum.update(data)
 457                     total_read += len(data)
 458             finally:
 459                 output.close()
 460             if self.hex_sum != checksum.hexdigest():
 461                 logging.warning('Bad checksum for %s fetched from %s.',
 462                                 self.name, url)
 463                 logging.warning('Got %s', checksum.hexdigest())
 464                 logging.warning('Expected %s', self.hex_sum)
 465                 os.unlink(local_path)
 466                 continue
 467             logging.info('Good checksum.')
 468             self.verified_package = local_path
 469             return True
 470         else:
 471             return False
 472
 473
 474 # NOTE: This class definition must come -before- all other ExternalPackage
 475 # classes that need to use this version of setuptools so that is is inserted
 476 # into the ExternalPackage.subclasses list before them.
 477 class SetuptoolsPackage(ExternalPackage):
 478     # For all known setuptools releases a string compare works for the
 479     # version string.  Hopefully they never release a 0.10.  (Their own
 480     # version comparison code would break if they did.)
 481     version = '0.6c11'
 482     urls = ('http://pypi.python.org/packages/source/s/setuptools/'
 483             'setuptools-%s.tar.gz' % (version,),)
 484     local_filename = 'setuptools-%s.tar.gz' % version
 485     hex_sum = '8d1ad6384d358c547c50c60f1bfdb3362c6c4a7d'
 486
 487     SUDO_SLEEP_DELAY = 15
 488
 489
 490     def _build_and_install(self, install_dir):
 491         """Install setuptools on the system."""
 492         logging.info('NOTE: setuptools install does not use install_dir.')
 493         return self._build_and_install_from_package(install_dir)
 494
 495
 496     def _build_and_install_current_dir(self, install_dir):
 497         egg_path = self._build_egg_using_setup_py()
 498         if not egg_path:
 499             return False
 500
 501         print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'
 502         print 'About to run sudo to install setuptools', self.version
 503         print 'on your system for use by', sys.executable, '\n'
 504         print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n'
 505         time.sleep(self.SUDO_SLEEP_DELAY)
 506
 507         # Copy the egg to the local filesystem /var/tmp so that root can
 508         # access it properly (avoid NFS squashroot issues).
 509         temp_dir = self._get_temp_dir()
 510         try:
 511             shutil.copy(egg_path, temp_dir)
 512             egg_name = os.path.split(egg_path)[1]
 513             temp_egg = os.path.join(temp_dir, egg_name)
 514             p = subprocess.Popen(['sudo', '/bin/sh', temp_egg],
 515                                  stdout=subprocess.PIPE)
 516             regex = re.compile('Copying (.*?) to (.*?)\n')
 517             match = regex.search(p.communicate()[0])
 518             status = p.wait()
 519
 520             if match:
 521                 compiled = os.path.join(match.group(2), match.group(1))
 522                 os.system("sudo chmod a+r '%s'" % compiled)
 523         finally:
 524             shutil.rmtree(temp_dir)
 525
 526         if status:
 527             logging.error('install of setuptools from egg failed.')
 528             return False
 529         return True
 530
 531
 532 class MySQLdbPackage(ExternalPackage):
 533     module_name = 'MySQLdb'
 534     version = '1.2.2'
 535     urls = ('http://downloads.sourceforge.net/project/mysql-python/'
 536             'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz'
 537             % dict(version=version),)
 538     local_filename = 'MySQL-python-%s.tar.gz' % version
 539     hex_sum = '945a04773f30091ad81743f9eb0329a3ee3de383'
 540
 541     _build_and_install_current_dir = (
 542             ExternalPackage._build_and_install_current_dir_setup_py)
 543
 544
 545     def _build_and_install(self, install_dir):
 546         if not os.path.exists('/usr/bin/mysql_config'):
 547             logging.error('You need to install /usr/bin/mysql_config')
 548             logging.error('On Ubuntu or Debian based systems use this: '
 549                           'sudo apt-get install libmysqlclient15-dev')
 550             return False
 551         return self._build_and_install_from_package(install_dir)
 552
 553
 554 class DjangoPackage(ExternalPackage):
 555     version = '1.1.1'
 556     local_filename = 'Django-%s.tar.gz' % version
 557     urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,)
 558     hex_sum = '441c54f0e90730bf4a55432b64519169b1e6ef20'
 559
 560     _build_and_install = ExternalPackage._build_and_install_from_package
 561     _build_and_install_current_dir = (
 562             ExternalPackage._build_and_install_current_dir_noegg)
 563
 564
 565     def _get_installed_version_from_module(self, module):
 566         try:
 567             return module.get_version().split()[0]
 568         except AttributeError:
 569             return '0.9.6'
 570
 571
 572
 573 class NumpyPackage(ExternalPackage):
 574     version = '1.2.1'
 575     local_filename = 'numpy-%s.tar.gz' % version
 576     urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/'
 577             'numpy-%(version)s.tar.gz' % dict(version=version),)
 578     hex_sum = '1aa706e733aea18eaffa70d93c0105718acb66c5'
 579
 580     _build_and_install = ExternalPackage._build_and_install_from_package
 581     _build_and_install_current_dir = (
 582             ExternalPackage._build_and_install_current_dir_setupegg_py)
 583
 584
 585 # This requires numpy so it must be declared after numpy to guarantee that it
 586 # is already installed.
 587 class MatplotlibPackage(ExternalPackage):
 588     version = '0.98.5.3'
 589     short_version = '0.98.5'
 590     local_filename = 'matplotlib-%s.tar.gz' % version
 591     urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/'
 592             'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),)
 593     hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6'
 594     os_requirements = {'/usr/include/ft2build.h': 'libfreetype6-dev',
 595                        '/usr/include/png.h': 'libpng12-dev'}
 596
 597     _build_and_install = ExternalPackage._build_and_install_from_package
 598     _build_and_install_current_dir = (
 599             ExternalPackage._build_and_install_current_dir_setupegg_py)
 600
 601
 602 class AtForkPackage(ExternalPackage):
 603     version = '0.1.2'
 604     local_filename = 'atfork-%s.zip' % version
 605     urls = ('http://python-atfork.googlecode.com/files/' + local_filename,)
 606     hex_sum = '5baa64c73e966b57fa797040585c760c502dc70b'
 607
 608     _build_and_install = ExternalPackage._build_and_install_from_package
 609     _build_and_install_current_dir = (
 610             ExternalPackage._build_and_install_current_dir_noegg)
 611
 612
 613 class ParamikoPackage(ExternalPackage):
 614     version = '1.7.5'
 615     local_filename = 'paramiko-%s.tar.gz' % version
 616     urls = ('http://www.lag.net/paramiko/download/' + local_filename,
 617             'ftp://mirrors.kernel.org/gentoo/distfiles/' + local_filename,)
 618     hex_sum = '592be7a08290070b71da63a8e6f28a803399e5c5'
 619
 620
 621     _build_and_install = ExternalPackage._build_and_install_from_package
 622
 623
 624     def _check_for_pycrypto(self):
 625         # NOTE(gps): Linux distros have better python-crypto packages than we
 626         # can easily get today via a wget due to the library's age and staleness
 627         # yet many security and behavior bugs are fixed by patches that distros
 628         # already apply.  PyCrypto has a new active maintainer in 2009.  Once a
 629         # new release is made (http://pycrypto.org/) we should add an installer.
 630         try:
 631             import Crypto
 632         except ImportError:
 633             logging.error('Please run "sudo apt-get install python-crypto" '
 634                           'or your Linux distro\'s equivalent.')
 635             return False
 636         return True
 637
 638
 639     def _build_and_install_current_dir(self, install_dir):
 640         if not self._check_for_pycrypto():
 641             return False
 642         # paramiko 1.7.4 doesn't require building, it is just a module directory
 643         # that we can rsync into place directly.
 644         if not os.path.isdir('paramiko'):
 645             raise Error('no paramiko directory in %s.' % os.getcwd())
 646         status = system("rsync -r 'paramiko' '%s/'" % install_dir)
 647         if status:
 648             logging.error('%s rsync to install_dir failed.' % self.name)
 649             return False
 650         return True
 651
 652
 653 class SimplejsonPackage(ExternalPackage):
 654     version = '2.0.9'
 655     local_filename = 'simplejson-%s.tar.gz' % version
 656     urls = ('http://pypi.python.org/packages/source/s/simplejson/' +
 657             local_filename,)
 658     hex_sum = 'b5b26059adbe677b06c299bed30557fcb0c7df8c'
 659
 660     _build_and_install = ExternalPackage._build_and_install_from_package
 661     _build_and_install_current_dir = (
 662                         ExternalPackage._build_and_install_current_dir_setup_py)
 663
 664
 665 class Httplib2Package(ExternalPackage):
 666     version = '0.6.0'
 667     local_filename = 'httplib2-%s.tar.gz' % version
 668     urls = ('http://httplib2.googlecode.com/files/' + local_filename,)
 669     hex_sum = '995344b2704826cc0d61a266e995b328d92445a5'
 670
 671     def _get_installed_version_from_module(self, module):
 672         # httplib2 doesn't contain a proper version
 673         return self.version
 674
 675     _build_and_install = ExternalPackage._build_and_install_from_package
 676     _build_and_install_current_dir = (
 677                         ExternalPackage._build_and_install_current_dir_noegg)
 678
 679
 680 class GwtPackage(ExternalPackage):
 681     """Fetch and extract a local copy of GWT used to build the frontend."""
 682
 683     version = '2.0.3'
 684     local_filename = 'gwt-%s.zip' % version
 685     urls = ('http://google-web-toolkit.googlecode.com/files/' + local_filename,)
 686     hex_sum = '1dabd25a02b9299f6fa84c51c97210a3373a663e'
 687     name = 'gwt'
 688     about_filename = 'about.txt'
 689     module_name = None  # Not a Python module.
 690
 691
 692     def is_needed(self, install_dir):
 693         gwt_dir = os.path.join(install_dir, self.name)
 694         about_file = os.path.join(install_dir, self.name, self.about_filename)
 695
 696         if not os.path.exists(gwt_dir) or not os.path.exists(about_file):
 697             logging.info('gwt not installed for autotest')
 698             return True
 699
 700         f = open(about_file, 'r')
 701         version_line = f.readline()
 702         f.close()
 703
 704         match = re.match(r'Google Web Toolkit (.*)', version_line)
 705         if not match:
 706             logging.info('did not find gwt version')
 707             return True
 708
 709         logging.info('found gwt version %s', match.group(1))
 710         return match.group(1) != self.version
 711
 712
 713     def _build_and_install(self, install_dir):
 714         os.chdir(install_dir)
 715         self._extract_compressed_package()
 716         extracted_dir = self.local_filename[:-len('.zip')]
 717         target_dir = os.path.join(install_dir, self.name)
 718         if os.path.exists(target_dir):
 719             shutil.rmtree(target_dir)
 720         os.rename(extracted_dir, target_dir)
 721         return True
 722
 723
 724 # This requires GWT to already be installed, so it must be declared after
 725 # GwtPackage
 726 class GwtIncubatorPackage(ExternalPackage):
 727     version = '20100204-r1747'
 728     local_filename = 'gwt-incubator-%s.jar' % version
 729     symlink_name = 'gwt-incubator.jar'
 730     urls = ('http://google-web-toolkit-incubator.googlecode.com/files/'
 731             + local_filename,)
 732     hex_sum = '0c9495634f0627d0b4de0d78a50a3aefebf67f8c'
 733     module_name = None  # Not a Python module
 734
 735
 736     def is_needed(self, install_dir):
 737         gwt_dir = os.path.join(install_dir, GwtPackage.name)
 738         return not os.path.exists(os.path.join(gwt_dir, self.local_filename))
 739
 740
 741     def _build_and_install(self, install_dir):
 742         dest = os.path.join(install_dir, GwtPackage.name, self.local_filename)
 743         shutil.copyfile(self.verified_package, dest)
 744
 745         symlink_path = os.path.join(
 746                 install_dir, GwtPackage.name, self.symlink_name)
 747         if os.path.exists(symlink_path):
 748             os.remove(symlink_path)
 749         os.symlink(dest, symlink_path)
 750         return True
 751
 752
 753 if __name__ == '__main__':
 754     sys.exit(main())