setup.py

   1 # Scrapy setup.py script
   2 #
   3 # It doesn't depend on setuptools, but if setuptools is available it'll use
   4 # some of its features, like package dependencies.
   5
   6 from __future__ import with_statement
   7
   8 from distutils.command.install_data import install_data
   9 from distutils.command.install import INSTALL_SCHEMES
  10 from subprocess import Popen, PIPE
  11 import os
  12 import sys
  13
  14 class osx_install_data(install_data):
  15     # On MacOS, the platform-specific lib dir is /System/Library/Framework/Python/.../
  16     # which is wrong. Python 2.5 supplied with MacOS 10.5 has an Apple-specific fix
  17     # for this in distutils.command.install_data#306. It fixes install_lib but not
  18     # install_data, which is why we roll our own install_data class.
  19
  20     def finalize_options(self):
  21         # By the time finalize_options is called, install.install_lib is set to the
  22         # fixed directory, so we set the installdir to install_lib. The
  23         # install_data class uses ('install_data', 'install_dir') instead.
  24         self.set_undefined_options('install', ('install_lib', 'install_dir'))
  25         install_data.finalize_options(self)
  26
  27 if sys.platform == "darwin":
  28     cmdclasses = {'install_data': osx_install_data}
  29 else:
  30     cmdclasses = {'install_data': install_data}
  31
  32 def fullsplit(path, result=None):
  33     """
  34     Split a pathname into components (the opposite of os.path.join) in a
  35     platform-neutral way.
  36     """
  37     if result is None:
  38         result = []
  39     head, tail = os.path.split(path)
  40     if head == '':
  41         return [tail] + result
  42     if head == path:
  43         return result
  44     return fullsplit(head, [tail] + result)
  45
  46 # Tell distutils to put the data_files in platform-specific installation
  47 # locations. See here for an explanation:
  48 # http://groups.google.com/group/comp.lang.python/browse_thread/thread/35ec7b2fed36eaec/2105ee4d9e8042cb
  49 for scheme in INSTALL_SCHEMES.values():
  50     scheme['data'] = scheme['purelib']
  51
  52 # Compile the list of packages available, because distutils doesn't have
  53 # an easy way to do this.
  54 packages, data_files = [], []
  55 root_dir = os.path.dirname(__file__)
  56 if root_dir != '':
  57     os.chdir(root_dir)
  58
  59 def is_not_module(filename):
  60     return os.path.splitext(f)[1] not in ['.py', '.pyc', '.pyo']
  61
  62 for scrapy_dir in ['scrapy', 'scrapyd']:
  63     for dirpath, dirnames, filenames in os.walk(scrapy_dir):
  64         # Ignore dirnames that start with '.'
  65         for i, dirname in enumerate(dirnames):
  66             if dirname.startswith('.'): del dirnames[i]
  67         if '__init__.py' in filenames:
  68             packages.append('.'.join(fullsplit(dirpath)))
  69             data = [f for f in filenames if is_not_module(f)]
  70             if data:
  71                 data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]])
  72         elif filenames:
  73             data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]])
  74
  75 # Small hack for working with bdist_wininst.
  76 # See http://mail.python.org/pipermail/distutils-sig/2004-August/004134.html
  77 if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst':
  78     for file_info in data_files:
  79         file_info[0] = '\\PURELIB\\%s' % file_info[0]
  80
  81 scripts = ['bin/scrapy']
  82 if os.name == 'nt':
  83     scripts.append('extras/scrapy.bat')
  84
  85 if os.environ.get('SCRAPY_VERSION_FROM_HG'):
  86     rev = Popen(["hg", "tip", "--template", "{rev}"], stdout=PIPE).communicate()[0]
  87     with open('scrapy/__init__.py', 'a') as f:
  88         f.write("\n__version__ = '.'.join(map(str, version_info)) + '.%s'" % rev)
  89 elif os.environ.get('SCRAPY_VERSION_FROM_GIT'):
  90     rev = Popen("git log --oneline | wc -l", shell=True, stdout=PIPE).communicate()[0]
  91     with open('scrapy/__init__.py', 'a') as f:
  92         f.write("\n__version__ = '.'.join(map(str, version_info)) + '.%s'" % rev.strip())
  93 version = __import__('scrapy').__version__
  94
  95 setup_args = {
  96     'name': 'Scrapy',
  97     'version': version,
  98     'url': 'http://scrapy.org',
  99     'description': 'A high-level Python Screen Scraping framework',
 100     'long_description': 'Scrapy is a high level scraping and web crawling framework for writing spiders to crawl and parse web pages for all kinds of purposes, from information retrieval to monitoring or testing web sites.',
 101     'author': 'Scrapy developers',
 102     'maintainer': 'Pablo Hoffman',
 103     'maintainer_email': 'pablo@pablohoffman.com',
 104     'license': 'BSD',
 105     'packages': packages,
 106     'cmdclass': cmdclasses,
 107     'data_files': data_files,
 108     'scripts': scripts,
 109     'classifiers': [
 110         'Programming Language :: Python',
 111         'Programming Language :: Python :: 2.5',
 112         'Programming Language :: Python :: 2.6',
 113         'Programming Language :: Python :: 2.7',
 114         'License :: OSI Approved :: BSD License',
 115         'Operating System :: OS Independent',
 116         'Development Status :: 5 - Production/Stable',
 117         'Intended Audience :: Developers',
 118         'Environment :: Console',
 119         'Topic :: Software Development :: Libraries :: Application Frameworks',
 120         'Topic :: Software Development :: Libraries :: Python Modules',
 121         'Topic :: Internet :: WWW/HTTP',
 122     ]
 123 }
 124
 125 try:
 126     from setuptools import setup
 127 except ImportError:
 128     from distutils.core import setup
 129 else:
 130     setup_args['install_requires'] = ['Twisted>=2.5', 'w3lib', 'pyOpenSSL']
 131     if sys.version_info < (2, 6):
 132         setup_args['install_requires'] += ['simplejson']
 133     try:
 134         import libxml2
 135     except ImportError:
 136         setup_args['install_requires'] += ['lxml']
 137
 138 setup(**setup_args)