buildtools/wafadmin/Tools/python.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2007 (ita)
   4 # Gustavo Carneiro (gjc), 2007
   5
   6 "Python support"
   7
   8 import os, sys
   9 import TaskGen, Utils, Options
  10 from Logs import debug, warn, info
  11 from TaskGen import extension, before, after, feature
  12 from Configure import conf
  13 from config_c import parse_flags
  14
  15 EXT_PY = ['.py']
  16 FRAG_2 = '''
  17 #include "Python.h"
  18 #ifdef __cplusplus
  19 extern "C" {
  20 #endif
  21         void Py_Initialize(void);
  22         void Py_Finalize(void);
  23 #ifdef __cplusplus
  24 }
  25 #endif
  26 int main()
  27 {
  28    Py_Initialize();
  29    Py_Finalize();
  30    return 0;
  31 }
  32 '''
  33
  34 @feature('pyext')
  35 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars', 'apply_bundle')
  36 @after('vars_target_cshlib')
  37 def init_pyext(self):
  38         self.default_install_path = '${PYTHONARCHDIR}'
  39         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  40         if not 'PYEXT' in self.uselib:
  41                 self.uselib.append('PYEXT')
  42         self.env['MACBUNDLE'] = True
  43
  44 @before('apply_link', 'apply_lib_vars', 'apply_type_vars')
  45 @after('apply_bundle')
  46 @feature('pyext')
  47 def pyext_shlib_ext(self):
  48         # override shlib_PATTERN set by the osx module
  49         self.env['shlib_PATTERN'] = self.env['pyext_PATTERN']
  50
  51 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars')
  52 @feature('pyembed')
  53 def init_pyembed(self):
  54         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  55         if not 'PYEMBED' in self.uselib:
  56                 self.uselib.append('PYEMBED')
  57
  58 @extension(EXT_PY)
  59 def process_py(self, node):
  60         if not (self.bld.is_install and self.install_path):
  61                 return
  62         def inst_py(ctx):
  63                 install_pyfile(self, node)
  64         self.bld.add_post_fun(inst_py)
  65
  66 def install_pyfile(self, node):
  67         path = self.bld.get_install_path(self.install_path + os.sep + node.name, self.env)
  68
  69         self.bld.install_files(self.install_path, [node], self.env, self.chmod, postpone=False)
  70         if self.bld.is_install < 0:
  71                 info("* removing byte compiled python files")
  72                 for x in 'co':
  73                         try:
  74                                 os.remove(path + x)
  75                         except OSError:
  76                                 pass
  77
  78         if self.bld.is_install > 0:
  79                 if self.env['PYC'] or self.env['PYO']:
  80                         info("* byte compiling %r" % path)
  81
  82                 if self.env['PYC']:
  83                         program = ("""
  84 import sys, py_compile
  85 for pyfile in sys.argv[1:]:
  86         py_compile.compile(pyfile, pyfile + 'c')
  87 """)
  88                         argv = [self.env['PYTHON'], '-c', program, path]
  89                         ret = Utils.pproc.Popen(argv).wait()
  90                         if ret:
  91                                 raise Utils.WafError('bytecode compilation failed %r' % path)
  92
  93                 if self.env['PYO']:
  94                         program = ("""
  95 import sys, py_compile
  96 for pyfile in sys.argv[1:]:
  97         py_compile.compile(pyfile, pyfile + 'o')
  98 """)
  99                         argv = [self.env['PYTHON'], self.env['PYFLAGS_OPT'], '-c', program, path]
 100                         ret = Utils.pproc.Popen(argv).wait()
 101                         if ret:
 102                                 raise Utils.WafError('bytecode compilation failed %r' % path)
 103
 104 # COMPAT
 105 class py_taskgen(TaskGen.task_gen):
 106         def __init__(self, *k, **kw):
 107                 TaskGen.task_gen.__init__(self, *k, **kw)
 108
 109 @before('apply_core')
 110 @after('vars_target_cprogram', 'vars_target_cshlib')
 111 @feature('py')
 112 def init_py(self):
 113         self.default_install_path = '${PYTHONDIR}'
 114
 115 def _get_python_variables(python_exe, variables, imports=['import sys']):
 116         """Run a python interpreter and print some variables"""
 117         program = list(imports)
 118         program.append('')
 119         for v in variables:
 120                 program.append("print(repr(%s))" % v)
 121         os_env = dict(os.environ)
 122         try:
 123                 del os_env['MACOSX_DEPLOYMENT_TARGET'] # see comments in the OSX tool
 124         except KeyError:
 125                 pass
 126         proc = Utils.pproc.Popen([python_exe, "-c", '\n'.join(program)], stdout=Utils.pproc.PIPE, env=os_env)
 127         output = proc.communicate()[0].split("\n") # do not touch, python3
 128         if proc.returncode:
 129                 if Options.options.verbose:
 130                         warn("Python program to extract python configuration variables failed:\n%s"
 131                                        % '\n'.join(["line %03i: %s" % (lineno+1, line) for lineno, line in enumerate(program)]))
 132                 raise RuntimeError
 133         return_values = []
 134         for s in output:
 135                 s = s.strip()
 136                 if not s:
 137                         continue
 138                 if s == 'None':
 139                         return_values.append(None)
 140                 elif (s[0] == "'" and s[-1] == "'") or (s[0] == '"' and s[-1] == '"'):
 141                         return_values.append(eval(s))
 142                 elif s[0].isdigit():
 143                         return_values.append(int(s))
 144                 else: break
 145         return return_values
 146
 147 @conf
 148 def check_python_headers(conf, mandatory=True):
 149         """Check for headers and libraries necessary to extend or embed python.
 150
 151         On success the environment variables xxx_PYEXT and xxx_PYEMBED are added for uselib
 152
 153         PYEXT: for compiling python extensions
 154         PYEMBED: for embedding a python interpreter"""
 155
 156         if not conf.env['CC_NAME'] and not conf.env['CXX_NAME']:
 157                 conf.fatal('load a compiler first (gcc, g++, ..)')
 158
 159         if not conf.env['PYTHON_VERSION']:
 160                 conf.check_python_version()
 161
 162         env = conf.env
 163         python = env['PYTHON']
 164         if not python:
 165                 conf.fatal('could not find the python executable')
 166
 167         ## On Mac OSX we need to use mac bundles for python plugins
 168         if Options.platform == 'darwin':
 169                 conf.check_tool('osx')
 170
 171         try:
 172                 # Get some python configuration variables using distutils
 173                 v = 'prefix SO SYSLIBS LDFLAGS SHLIBS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET'.split()
 174                 (python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 175                  python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED,
 176                  python_MACOSX_DEPLOYMENT_TARGET) = \
 177                         _get_python_variables(python, ["get_config_var('%s') or ''" % x for x in v],
 178                                               ['from distutils.sysconfig import get_config_var'])
 179         except RuntimeError:
 180                 conf.fatal("Python development headers not found (-v for details).")
 181
 182         conf.log.write("""Configuration returned from %r:
 183 python_prefix = %r
 184 python_SO = %r
 185 python_SYSLIBS = %r
 186 python_LDFLAGS = %r
 187 python_SHLIBS = %r
 188 python_LIBDIR = %r
 189 python_LIBPL = %r
 190 INCLUDEPY = %r
 191 Py_ENABLE_SHARED = %r
 192 MACOSX_DEPLOYMENT_TARGET = %r
 193 """ % (python, python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 194         python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED, python_MACOSX_DEPLOYMENT_TARGET))
 195
 196         # Allow some python overrides from env vars for cross-compiling
 197         os_env = dict(os.environ)
 198
 199         override_python_LDFLAGS = os_env.get('python_LDFLAGS', None)
 200         if override_python_LDFLAGS is not None:
 201                 conf.log.write("python_LDFLAGS override from environment = %r\n" % (override_python_LDFLAGS))
 202                 python_LDFLAGS = override_python_LDFLAGS
 203
 204         override_python_LIBDIR = os_env.get('python_LIBDIR', None)
 205         if override_python_LIBDIR is not None:
 206                 conf.log.write("python_LIBDIR override from environment = %r\n" % (override_python_LIBDIR))
 207                 python_LIBDIR = override_python_LIBDIR
 208
 209         if python_MACOSX_DEPLOYMENT_TARGET:
 210                 conf.env['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 211                 conf.environ['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 212
 213         env['pyext_PATTERN'] = '%s'+python_SO
 214
 215         # Check for python libraries for embedding
 216         if python_SYSLIBS is not None:
 217                 for lib in python_SYSLIBS.split():
 218                         if lib.startswith('-l'):
 219                                 lib = lib[2:] # strip '-l'
 220                         env.append_value('LIB_PYEMBED', lib)
 221
 222         if python_SHLIBS is not None:
 223                 for lib in python_SHLIBS.split():
 224                         if lib.startswith('-l'):
 225                                 env.append_value('LIB_PYEMBED', lib[2:]) # strip '-l'
 226                         else:
 227                                 env.append_value('LINKFLAGS_PYEMBED', lib)
 228
 229         if Options.platform != 'darwin' and python_LDFLAGS:
 230                 parse_flags(python_LDFLAGS, 'PYEMBED', env)
 231
 232         result = False
 233         name = 'python' + env['PYTHON_VERSION']
 234
 235         if python_LIBDIR is not None:
 236                 path = [python_LIBDIR]
 237                 conf.log.write("\n\n# Trying LIBDIR: %r\n" % path)
 238                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 239
 240         if not result and python_LIBPL is not None:
 241                 conf.log.write("\n\n# try again with -L$python_LIBPL (some systems don't install the python library in $prefix/lib)\n")
 242                 path = [python_LIBPL]
 243                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 244
 245         if not result:
 246                 conf.log.write("\n\n# try again with -L$prefix/libs, and pythonXY name rather than pythonX.Y (win32)\n")
 247                 path = [os.path.join(python_prefix, "libs")]
 248                 name = 'python' + env['PYTHON_VERSION'].replace('.', '')
 249                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 250
 251         if result:
 252                 env['LIBPATH_PYEMBED'] = path
 253                 env.append_value('LIB_PYEMBED', name)
 254         else:
 255                 conf.log.write("\n\n### LIB NOT FOUND\n")
 256
 257         # under certain conditions, python extensions must link to
 258         # python libraries, not just python embedding programs.
 259         if (sys.platform == 'win32' or sys.platform.startswith('os2')
 260                 or sys.platform == 'darwin' or Py_ENABLE_SHARED):
 261                 env['LIBPATH_PYEXT'] = env['LIBPATH_PYEMBED']
 262                 env['LIB_PYEXT'] = env['LIB_PYEMBED']
 263
 264         # We check that pythonX.Y-config exists, and if it exists we
 265         # use it to get only the includes, else fall back to distutils.
 266         python_config = conf.find_program(
 267                 'python%s-config' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 268                 var='PYTHON_CONFIG')
 269         if not python_config:
 270                 python_config = conf.find_program(
 271                         'python-config-%s' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 272                         var='PYTHON_CONFIG')
 273
 274         includes = []
 275         if python_config:
 276                 for incstr in Utils.cmd_output("%s --includes" % (python_config,)).strip().split():
 277                         # strip the -I or /I
 278                         if (incstr.startswith('-I')
 279                             or incstr.startswith('/I')):
 280                                 incstr = incstr[2:]
 281                         # append include path, unless already given
 282                         if incstr not in includes:
 283                                 includes.append(incstr)
 284                 conf.log.write("Include path for Python extensions "
 285                                "(found via python-config --includes): %r\n" % (includes,))
 286                 env['CPPPATH_PYEXT'] = includes
 287                 env['CPPPATH_PYEMBED'] = includes
 288         else:
 289                 conf.log.write("Include path for Python extensions "
 290                                "(found via distutils module): %r\n" % (INCLUDEPY,))
 291                 env['CPPPATH_PYEXT'] = [INCLUDEPY]
 292                 env['CPPPATH_PYEMBED'] = [INCLUDEPY]
 293
 294         # Code using the Python API needs to be compiled with -fno-strict-aliasing
 295         if env['CC_NAME'] == 'gcc':
 296                 env.append_value('CCFLAGS_PYEMBED', '-fno-strict-aliasing')
 297                 env.append_value('CCFLAGS_PYEXT', '-fno-strict-aliasing')
 298         if env['CXX_NAME'] == 'gcc':
 299                 env.append_value('CXXFLAGS_PYEMBED', '-fno-strict-aliasing')
 300                 env.append_value('CXXFLAGS_PYEXT', '-fno-strict-aliasing')
 301
 302         # See if it compiles
 303         conf.check(define_name='HAVE_PYTHON_H',
 304                    uselib='PYEMBED', fragment=FRAG_2,
 305                    errmsg='Could not find the python development headers', mandatory=mandatory)
 306
 307 @conf
 308 def check_python_version(conf, minver=None):
 309         """
 310         Check if the python interpreter is found matching a given minimum version.
 311         minver should be a tuple, eg. to check for python >= 2.4.2 pass (2,4,2) as minver.
 312
 313         If successful, PYTHON_VERSION is defined as 'MAJOR.MINOR'
 314         (eg. '2.4') of the actual python version found, and PYTHONDIR is
 315         defined, pointing to the site-packages directory appropriate for
 316         this python version, where modules/packages/extensions should be
 317         installed.
 318         """
 319         assert minver is None or isinstance(minver, tuple)
 320         python = conf.env['PYTHON']
 321         if not python:
 322                 conf.fatal('could not find the python executable')
 323
 324         # Get python version string
 325         cmd = [python, "-c", "import sys\nfor x in sys.version_info: print(str(x))"]
 326         debug('python: Running python command %r' % cmd)
 327         proc = Utils.pproc.Popen(cmd, stdout=Utils.pproc.PIPE, shell=False)
 328         lines = proc.communicate()[0].split()
 329         assert len(lines) == 5, "found %i lines, expected 5: %r" % (len(lines), lines)
 330         pyver_tuple = (int(lines[0]), int(lines[1]), int(lines[2]), lines[3], int(lines[4]))
 331
 332         # compare python version with the minimum required
 333         result = (minver is None) or (pyver_tuple >= minver)
 334
 335         if result:
 336                 # define useful environment variables
 337                 pyver = '.'.join([str(x) for x in pyver_tuple[:2]])
 338                 conf.env['PYTHON_VERSION'] = pyver
 339
 340                 if 'PYTHONDIR' in conf.environ:
 341                         pydir = conf.environ['PYTHONDIR']
 342                 else:
 343                         if sys.platform == 'win32':
 344                                 (python_LIBDEST, pydir) = \
 345                                                 _get_python_variables(python,
 346                                                                                           ["get_config_var('LIBDEST') or ''",
 347                                                                                            "get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 348                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 349                         else:
 350                                 python_LIBDEST = None
 351                                 (pydir,) = \
 352                                                 _get_python_variables(python,
 353                                                                                           ["get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 354                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 355                         if python_LIBDEST is None:
 356                                 if conf.env['LIBDIR']:
 357                                         python_LIBDEST = os.path.join(conf.env['LIBDIR'], "python" + pyver)
 358                                 else:
 359                                         python_LIBDEST = os.path.join(conf.env['PREFIX'], "lib", "python" + pyver)
 360
 361                 if 'PYTHONARCHDIR' in conf.environ:
 362                         pyarchdir = conf.environ['PYTHONARCHDIR']
 363                 else:
 364                         (pyarchdir,) = _get_python_variables(python,
 365                                                                                         ["get_python_lib(plat_specific=1, standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 366                                                                                         ['from distutils.sysconfig import get_config_var, get_python_lib'])
 367                         if not pyarchdir:
 368                                 pyarchdir = pydir
 369
 370                 if hasattr(conf, 'define'): # conf.define is added by the C tool, so may not exist
 371                         conf.define('PYTHONDIR', pydir)
 372                         conf.define('PYTHONARCHDIR', pyarchdir)
 373
 374                 conf.env['PYTHONDIR'] = pydir
 375
 376         # Feedback
 377         pyver_full = '.'.join(map(str, pyver_tuple[:3]))
 378         if minver is None:
 379                 conf.check_message_custom('Python version', '', pyver_full)
 380         else:
 381                 minver_str = '.'.join(map(str, minver))
 382                 conf.check_message('Python version', ">= %s" % minver_str, result, option=pyver_full)
 383
 384         if not result:
 385                 conf.fatal('The python version is too old (%r)' % pyver_full)
 386
 387 @conf
 388 def check_python_module(conf, module_name):
 389         """
 390         Check if the selected python interpreter can import the given python module.
 391         """
 392         result = not Utils.pproc.Popen([conf.env['PYTHON'], "-c", "import %s" % module_name],
 393                            stderr=Utils.pproc.PIPE, stdout=Utils.pproc.PIPE).wait()
 394         conf.check_message('Python module', module_name, result)
 395         if not result:
 396                 conf.fatal('Could not find the python module %r' % module_name)
 397
 398 def detect(conf):
 399
 400         if not conf.env.PYTHON:
 401                 conf.env.PYTHON = sys.executable
 402
 403         python = conf.find_program('python', var='PYTHON')
 404         if not python:
 405                 conf.fatal('Could not find the path of the python executable')
 406
 407         if conf.env.PYTHON != sys.executable:
 408                 warn("python executable '%s' different from sys.executable '%s'" % (conf.env.PYTHON, sys.executable))
 409
 410         v = conf.env
 411         v['PYCMD'] = '"import sys, py_compile;py_compile.compile(sys.argv[1], sys.argv[2])"'
 412         v['PYFLAGS'] = ''
 413         v['PYFLAGS_OPT'] = '-O'
 414
 415         v['PYC'] = getattr(Options.options, 'pyc', 1)
 416         v['PYO'] = getattr(Options.options, 'pyo', 1)
 417
 418 def set_options(opt):
 419         opt.add_option('--nopyc',
 420                         action='store_false',
 421                         default=1,
 422                         help = 'Do not install bytecode compiled .pyc files (configuration) [Default:install]',
 423                         dest = 'pyc')
 424         opt.add_option('--nopyo',
 425                         action='store_false',
 426                         default=1,
 427                         help='Do not install optimised compiled .pyo files (configuration) [Default:install]',
 428                         dest='pyo')
 429