3 # Thomas Nagy, 2007-2015 (ita)
4 # Gustavo Carneiro (gjc), 2007
7 Support for Python, detect the headers and libraries and provide
8 *use* variables to link C/C++ programs against them::
11 opt.load('compiler_c python')
13 conf.load('compiler_c python')
14 conf.check_python_version((2,4,2))
15 conf.check_python_headers()
17 bld.program(features='pyembed', source='a.c', target='myprog')
18 bld.shlib(features='pyext', source='b.c', target='mylib')
22 from waflib import Errors, Logs, Node, Options, Task, Utils
23 from waflib.TaskGen import extension, before_method, after_method, feature
24 from waflib.Configure import conf
31 void Py_Initialize(void);
32 void Py_Finalize(void);
36 int main(int argc, char **argv)
38 (void)argc; (void)argv;
45 Piece of C/C++ code used in :py:func:`waflib.Tools.python.check_python_headers`
49 import sys, py_compile
50 py_compile.compile(sys.argv[1], sys.argv[2], sys.argv[3], True)
53 Piece of Python code used in :py:class:`waflib.Tools.python.pyo` and :py:class:`waflib.Tools.python.pyc` for byte-compiling python files
56 DISTUTILS_IMP = ['from distutils.sysconfig import get_config_var, get_python_lib']
58 @before_method('process_source')
62 Create tasks to byte-compile .py files and install them, if requested
64 self.install_path = getattr(self, 'install_path', '${PYTHONDIR}')
65 install_from = getattr(self, 'install_from', None)
66 if install_from and not isinstance(install_from, Node.Node):
67 install_from = self.path.find_dir(install_from)
68 self.install_from = install_from
70 ver = self.env.PYTHON_VERSION
72 self.bld.fatal('Installing python files requires PYTHON_VERSION, try conf.check_python_version')
74 if int(ver.replace('.', '')) > 31:
75 self.install_32 = True
78 def process_py(self, node):
80 Add signature of .py file, so it will be byte-compiled when necessary
82 assert(hasattr(self, 'install_path')), 'add features="py"'
84 # where to install the python file
87 self.add_install_files(install_to=self.install_path, install_from=node, cwd=self.install_from, relative_trick=True)
89 self.add_install_files(install_to=self.install_path, install_from=node, relative_trick=True)
99 pyd = Utils.subst_vars("%s/%s" % (self.install_path, node.path_from(self.install_from)), self.env)
101 pyd = Utils.subst_vars("%s/%s" % (self.install_path, node.path_from(self.path)), self.env)
106 if self.env.PYTAG and not self.env.NOPYCACHE:
107 # __pycache__ installation for python 3.2 - PEP 3147
108 name = node.name[:-3]
109 pyobj = node.parent.get_bld().make_node('__pycache__').make_node("%s.%s.%s" % (name, self.env.PYTAG, ext))
112 pyobj = node.change_ext(".%s" % ext)
114 tsk = self.create_task(ext, node, pyobj)
117 if self.install_path:
118 self.add_install_files(install_to=os.path.dirname(pyd), install_from=pyobj, cwd=node.parent.get_bld(), relative_trick=True)
120 class pyc(Task.Task):
122 Byte-compiling python files
126 node = self.outputs[0]
127 return node.path_from(node.ctx.launch_node())
129 cmd = [Utils.subst_vars('${PYTHON}', self.env), '-c', INST, self.inputs[0].abspath(), self.outputs[0].abspath(), self.pyd]
130 ret = self.generator.bld.exec_command(cmd)
133 class pyo(Task.Task):
135 Byte-compiling python files
139 node = self.outputs[0]
140 return node.path_from(node.ctx.launch_node())
142 cmd = [Utils.subst_vars('${PYTHON}', self.env), Utils.subst_vars('${PYFLAGS_OPT}', self.env), '-c', INST, self.inputs[0].abspath(), self.outputs[0].abspath(), self.pyd]
143 ret = self.generator.bld.exec_command(cmd)
147 @before_method('propagate_uselib_vars', 'apply_link')
148 @after_method('apply_bundle')
149 def init_pyext(self):
151 Change the values of *cshlib_PATTERN* and *cxxshlib_PATTERN* to remove the
152 *lib* prefix from library names.
154 self.uselib = self.to_list(getattr(self, 'uselib', []))
155 if not 'PYEXT' in self.uselib:
156 self.uselib.append('PYEXT')
157 # override shlib_PATTERN set by the osx module
158 self.env.cshlib_PATTERN = self.env.cxxshlib_PATTERN = self.env.macbundle_PATTERN = self.env.pyext_PATTERN
159 self.env.fcshlib_PATTERN = self.env.dshlib_PATTERN = self.env.pyext_PATTERN
162 if not self.install_path:
164 except AttributeError:
165 self.install_path = '${PYTHONARCHDIR}'
168 @before_method('apply_link', 'apply_bundle')
169 def set_bundle(self):
170 """Mac-specific pyext extension that enables bundles from c_osx.py"""
171 if Utils.unversioned_sys_platform() == 'darwin':
172 self.mac_bundle = True
174 @before_method('propagate_uselib_vars')
176 def init_pyembed(self):
178 Add the PYEMBED variable.
180 self.uselib = self.to_list(getattr(self, 'uselib', []))
181 if not 'PYEMBED' in self.uselib:
182 self.uselib.append('PYEMBED')
185 def get_python_variables(self, variables, imports=None):
187 Spawn a new python process to dump configuration variables
189 :param variables: variables to print
190 :type variables: list of string
191 :param imports: one import by element
192 :type imports: list of string
193 :return: the variable values
194 :rtype: list of string
198 imports = self.python_imports
199 except AttributeError:
200 imports = DISTUTILS_IMP
202 program = list(imports) # copy
205 program.append("print(repr(%s))" % v)
206 os_env = dict(os.environ)
208 del os_env['MACOSX_DEPLOYMENT_TARGET'] # see comments in the OSX tool
213 out = self.cmd_and_log(self.env.PYTHON + ['-c', '\n'.join(program)], env=os_env)
214 except Errors.WafError:
215 self.fatal('The distutils module is unusable: install "python-devel"?')
218 for s in out.splitlines():
223 return_values.append(None)
224 elif (s[0] == "'" and s[-1] == "'") or (s[0] == '"' and s[-1] == '"'):
225 return_values.append(eval(s))
227 return_values.append(int(s))
232 def test_pyembed(self, mode, msg='Testing pyembed configuration'):
233 self.check(header_name='Python.h', define_name='HAVE_PYEMBED', msg=msg,
234 fragment=FRAG, errmsg='Could not build a python embedded interpreter',
235 features='%s %sprogram pyembed' % (mode, mode))
238 def test_pyext(self, mode, msg='Testing pyext configuration'):
239 self.check(header_name='Python.h', define_name='HAVE_PYEXT', msg=msg,
240 fragment=FRAG, errmsg='Could not build python extensions',
241 features='%s %sshlib pyext' % (mode, mode))
244 def python_cross_compile(self, features='pyembed pyext'):
246 For cross-compilation purposes, it is possible to bypass the normal detection and set the flags that you want:
247 PYTHON_VERSION='3.4' PYTAG='cpython34' pyext_PATTERN="%s.so" PYTHON_LDFLAGS='-lpthread -ldl' waf configure
249 The following variables are used:
250 PYTHON_VERSION required
252 PYTHON_LDFLAGS required
253 pyext_PATTERN required
255 PYTHON_PYEMBED_LDFLAGS
257 features = Utils.to_list(features)
258 if not ('PYTHON_LDFLAGS' in self.environ or 'PYTHON_PYEXT_LDFLAGS' in self.environ or 'PYTHON_PYEMBED_LDFLAGS' in self.environ):
261 for x in 'PYTHON_VERSION PYTAG pyext_PATTERN'.split():
262 if not x in self.environ:
263 self.fatal('Please set %s in the os environment' % x)
265 self.env[x] = self.environ[x]
267 xx = self.env.CXX_NAME and 'cxx' or 'c'
268 if 'pyext' in features:
269 flags = self.environ.get('PYTHON_PYEXT_LDFLAGS', self.environ.get('PYTHON_LDFLAGS'))
271 self.fatal('No flags provided through PYTHON_PYEXT_LDFLAGS as required')
273 self.parse_flags(flags, 'PYEXT')
275 if 'pyembed' in features:
276 flags = self.environ.get('PYTHON_PYEMBED_LDFLAGS', self.environ.get('PYTHON_LDFLAGS'))
278 self.fatal('No flags provided through PYTHON_PYEMBED_LDFLAGS as required')
280 self.parse_flags(flags, 'PYEMBED')
281 self.test_pyembed(xx)
285 def check_python_headers(conf, features='pyembed pyext'):
287 Check for headers and libraries necessary to extend or embed python by using the module *distutils*.
288 On success the environment variables xxx_PYEXT and xxx_PYEMBED are added:
290 * PYEXT: for compiling python extensions
291 * PYEMBED: for embedding a python interpreter
293 features = Utils.to_list(features)
294 assert ('pyembed' in features) or ('pyext' in features), "check_python_headers features must include 'pyembed' and/or 'pyext'"
296 if not env.CC_NAME and not env.CXX_NAME:
297 conf.fatal('load a compiler first (gcc, g++, ..)')
299 # bypass all the code below for cross-compilation
300 if conf.python_cross_compile(features):
303 if not env.PYTHON_VERSION:
304 conf.check_python_version()
308 conf.fatal('Could not find the python executable')
310 # so we actually do all this for compatibility reasons and for obtaining pyext_PATTERN below
311 v = 'prefix SO LDFLAGS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET LDSHARED CFLAGS LDVERSION'.split()
313 lst = conf.get_python_variables(["get_config_var('%s') or ''" % x for x in v])
315 conf.fatal("Python development headers not found (-v for details).")
317 vals = ['%s = %r' % (x, y) for (x, y) in zip(v, lst)]
318 conf.to_log("Configuration returned from %r:\n%s\n" % (pybin, '\n'.join(vals)))
320 dct = dict(zip(v, lst))
321 x = 'MACOSX_DEPLOYMENT_TARGET'
323 env[x] = conf.environ[x] = dct[x]
324 env.pyext_PATTERN = '%s' + dct['SO'] # not a mistake
327 # Try to get pythonX.Y-config
328 num = '.'.join(env.PYTHON_VERSION.split('.')[:2])
329 conf.find_program([''.join(pybin) + '-config', 'python%s-config' % num, 'python-config-%s' % num, 'python%sm-config' % num], var='PYTHON_CONFIG', msg="python-config", mandatory=False)
331 if env.PYTHON_CONFIG:
332 # check python-config output only once
333 if conf.env.HAVE_PYTHON_H:
336 # python2.6-config requires 3 runs
337 all_flags = [['--cflags', '--libs', '--ldflags']]
338 if sys.hexversion < 0x2070000:
339 all_flags = [[k] for k in all_flags[0]]
341 xx = env.CXX_NAME and 'cxx' or 'c'
343 if 'pyembed' in features:
344 for flags in all_flags:
345 # Python 3.8 has different flags for pyembed, needs --embed
346 embedflags = flags + ['--embed']
348 conf.check_cfg(msg='Asking python-config for pyembed %r flags' % ' '.join(embedflags), path=env.PYTHON_CONFIG, package='', uselib_store='PYEMBED', args=embedflags)
349 except conf.errors.ConfigurationError:
350 # However Python < 3.8 doesn't accept --embed, so we need a fallback
351 conf.check_cfg(msg='Asking python-config for pyembed %r flags' % ' '.join(flags), path=env.PYTHON_CONFIG, package='', uselib_store='PYEMBED', args=flags)
354 conf.test_pyembed(xx)
355 except conf.errors.ConfigurationError:
357 if dct['Py_ENABLE_SHARED'] and dct['LIBDIR']:
358 env.append_unique('LIBPATH_PYEMBED', [dct['LIBDIR']])
359 conf.test_pyembed(xx)
363 if 'pyext' in features:
364 for flags in all_flags:
365 conf.check_cfg(msg='Asking python-config for pyext %r flags' % ' '.join(flags), path=env.PYTHON_CONFIG, package='', uselib_store='PYEXT', args=flags)
369 except conf.errors.ConfigurationError:
371 if dct['Py_ENABLE_SHARED'] and dct['LIBDIR']:
372 env.append_unique('LIBPATH_PYEXT', [dct['LIBDIR']])
377 conf.define('HAVE_PYTHON_H', 1)
380 # No python-config, do something else on windows systems
381 all_flags = dct['LDFLAGS'] + ' ' + dct['CFLAGS']
382 conf.parse_flags(all_flags, 'PYEMBED')
384 all_flags = dct['LDFLAGS'] + ' ' + dct['LDSHARED'] + ' ' + dct['CFLAGS']
385 conf.parse_flags(all_flags, 'PYEXT')
388 if not dct["LDVERSION"]:
389 dct["LDVERSION"] = env.PYTHON_VERSION
391 # further simplification will be complicated
392 for name in ('python' + dct['LDVERSION'], 'python' + env.PYTHON_VERSION + 'm', 'python' + env.PYTHON_VERSION.replace('.', '')):
394 # LIBPATH_PYEMBED is already set; see if it works.
395 if not result and env.LIBPATH_PYEMBED:
396 path = env.LIBPATH_PYEMBED
397 conf.to_log("\n\n# Trying default LIBPATH_PYEMBED: %r\n" % path)
398 result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in LIBPATH_PYEMBED' % name)
400 if not result and dct['LIBDIR']:
401 path = [dct['LIBDIR']]
402 conf.to_log("\n\n# try again with -L$python_LIBDIR: %r\n" % path)
403 result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in LIBDIR' % name)
405 if not result and dct['LIBPL']:
406 path = [dct['LIBPL']]
407 conf.to_log("\n\n# try again with -L$python_LIBPL (some systems don't install the python library in $prefix/lib)\n")
408 result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in python_LIBPL' % name)
411 path = [os.path.join(dct['prefix'], "libs")]
412 conf.to_log("\n\n# try again with -L$prefix/libs, and pythonXY name rather than pythonX.Y (win32)\n")
413 result = conf.check(lib=name, uselib='PYEMBED', libpath=path, mandatory=False, msg='Checking for library %s in $prefix/libs' % name)
416 break # do not forget to set LIBPATH_PYEMBED
419 env.LIBPATH_PYEMBED = path
420 env.append_value('LIB_PYEMBED', [name])
422 conf.to_log("\n\n### LIB NOT FOUND\n")
424 # under certain conditions, python extensions must link to
425 # python libraries, not just python embedding programs.
426 if Utils.is_win32 or dct['Py_ENABLE_SHARED']:
427 env.LIBPATH_PYEXT = env.LIBPATH_PYEMBED
428 env.LIB_PYEXT = env.LIB_PYEMBED
430 conf.to_log("Include path for Python extensions (found via distutils module): %r\n" % (dct['INCLUDEPY'],))
431 env.INCLUDES_PYEXT = [dct['INCLUDEPY']]
432 env.INCLUDES_PYEMBED = [dct['INCLUDEPY']]
434 # Code using the Python API needs to be compiled with -fno-strict-aliasing
435 if env.CC_NAME == 'gcc':
436 env.append_value('CFLAGS_PYEMBED', ['-fno-strict-aliasing'])
437 env.append_value('CFLAGS_PYEXT', ['-fno-strict-aliasing'])
438 if env.CXX_NAME == 'gcc':
439 env.append_value('CXXFLAGS_PYEMBED', ['-fno-strict-aliasing'])
440 env.append_value('CXXFLAGS_PYEXT', ['-fno-strict-aliasing'])
442 if env.CC_NAME == "msvc":
443 from distutils.msvccompiler import MSVCCompiler
444 dist_compiler = MSVCCompiler()
445 dist_compiler.initialize()
446 env.append_value('CFLAGS_PYEXT', dist_compiler.compile_options)
447 env.append_value('CXXFLAGS_PYEXT', dist_compiler.compile_options)
448 env.append_value('LINKFLAGS_PYEXT', dist_compiler.ldflags_shared)
451 conf.check(header_name='Python.h', define_name='HAVE_PYTHON_H', uselib='PYEMBED', fragment=FRAG, errmsg='Distutils not installed? Broken python installation? Get python-config now!')
454 def check_python_version(conf, minver=None):
456 Check if the python interpreter is found matching a given minimum version.
457 minver should be a tuple, eg. to check for python >= 2.4.2 pass (2,4,2) as minver.
459 If successful, PYTHON_VERSION is defined as 'MAJOR.MINOR' (eg. '2.4')
460 of the actual python version found, and PYTHONDIR and PYTHONARCHDIR
461 are defined, pointing to the site-packages directories appropriate for
462 this python version, where modules/packages/extensions should be
465 :param minver: minimum version
466 :type minver: tuple of int
468 assert minver is None or isinstance(minver, tuple)
469 pybin = conf.env.PYTHON
471 conf.fatal('could not find the python executable')
473 # Get python version string
474 cmd = pybin + ['-c', 'import sys\nfor x in sys.version_info: print(str(x))']
475 Logs.debug('python: Running python command %r', cmd)
476 lines = conf.cmd_and_log(cmd).split()
477 assert len(lines) == 5, "found %r lines, expected 5: %r" % (len(lines), lines)
478 pyver_tuple = (int(lines[0]), int(lines[1]), int(lines[2]), lines[3], int(lines[4]))
480 # Compare python version with the minimum required
481 result = (minver is None) or (pyver_tuple >= minver)
484 # define useful environment variables
485 pyver = '.'.join([str(x) for x in pyver_tuple[:2]])
486 conf.env.PYTHON_VERSION = pyver
488 if 'PYTHONDIR' in conf.env:
489 # Check if --pythondir was specified
490 pydir = conf.env.PYTHONDIR
491 elif 'PYTHONDIR' in conf.environ:
492 # Check environment for PYTHONDIR
493 pydir = conf.environ['PYTHONDIR']
495 # Finally, try to guess
497 (python_LIBDEST, pydir) = conf.get_python_variables(
498 ["get_config_var('LIBDEST') or ''",
499 "get_python_lib(standard_lib=0) or ''"])
501 python_LIBDEST = None
502 (pydir,) = conf.get_python_variables( ["get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env.PREFIX])
503 if python_LIBDEST is None:
505 python_LIBDEST = os.path.join(conf.env.LIBDIR, 'python' + pyver)
507 python_LIBDEST = os.path.join(conf.env.PREFIX, 'lib', 'python' + pyver)
509 if 'PYTHONARCHDIR' in conf.env:
510 # Check if --pythonarchdir was specified
511 pyarchdir = conf.env.PYTHONARCHDIR
512 elif 'PYTHONARCHDIR' in conf.environ:
513 # Check environment for PYTHONDIR
514 pyarchdir = conf.environ['PYTHONARCHDIR']
516 # Finally, try to guess
517 (pyarchdir, ) = conf.get_python_variables( ["get_python_lib(plat_specific=1, standard_lib=0, prefix=%r) or ''" % conf.env.PREFIX])
521 if hasattr(conf, 'define'): # conf.define is added by the C tool, so may not exist
522 conf.define('PYTHONDIR', pydir)
523 conf.define('PYTHONARCHDIR', pyarchdir)
525 conf.env.PYTHONDIR = pydir
526 conf.env.PYTHONARCHDIR = pyarchdir
529 pyver_full = '.'.join(map(str, pyver_tuple[:3]))
531 conf.msg('Checking for python version', pyver_full)
533 minver_str = '.'.join(map(str, minver))
534 conf.msg('Checking for python version >= %s' % (minver_str,), pyver_full, color=result and 'GREEN' or 'YELLOW')
537 conf.fatal('The python version is too old, expecting %r' % (minver,))
539 PYTHON_MODULE_TEMPLATE = '''
540 import %s as current_module
541 version = getattr(current_module, '__version__', None)
542 if version is not None:
545 print('unknown version')
549 def check_python_module(conf, module_name, condition=''):
551 Check if the selected python interpreter can import the given python module::
554 conf.check_python_module('pygccxml')
555 conf.check_python_module('re', condition="ver > num(2, 0, 4) and ver <= num(3, 0, 0)")
557 :param module_name: module
558 :type module_name: string
560 msg = "Checking for python module %r" % module_name
562 msg = '%s (%s)' % (msg, condition)
565 ret = conf.cmd_and_log(conf.env.PYTHON + ['-c', PYTHON_MODULE_TEMPLATE % module_name])
566 except Errors.WafError:
568 conf.fatal('Could not find the python module %r' % module_name)
573 if ret == 'unknown version':
574 conf.fatal('Could not check the %s version' % module_name)
576 from distutils.version import LooseVersion
578 if isinstance(k[0], int):
579 return LooseVersion('.'.join([str(x) for x in k]))
581 return LooseVersion(k[0])
582 d = {'num': num, 'ver': LooseVersion(ret)}
583 ev = eval(condition, {}, d)
585 conf.fatal('The %s version does not satisfy the requirements' % module_name)
587 if ret == 'unknown version':
594 Detect the python interpreter
597 if getattr(Options.options, 'pythondir', None):
598 v.PYTHONDIR = Options.options.pythondir
599 if getattr(Options.options, 'pythonarchdir', None):
600 v.PYTHONARCHDIR = Options.options.pythonarchdir
601 if getattr(Options.options, 'nopycache', None):
602 v.NOPYCACHE=Options.options.nopycache
605 v.PYTHON = [getattr(Options.options, 'python', None) or sys.executable]
606 v.PYTHON = Utils.to_list(v.PYTHON)
607 conf.find_program('python', var='PYTHON')
612 v.PYC = getattr(Options.options, 'pyc', 1)
613 v.PYO = getattr(Options.options, 'pyo', 1)
616 v.PYTAG = conf.cmd_and_log(conf.env.PYTHON + ['-c', "import imp;print(imp.get_tag())"]).strip()
617 except Errors.WafError:
622 Add python-specific options
624 pyopt=opt.add_option_group("Python Options")
625 pyopt.add_option('--nopyc', dest = 'pyc', action='store_false', default=1,
626 help = 'Do not install bytecode compiled .pyc files (configuration) [Default:install]')
627 pyopt.add_option('--nopyo', dest='pyo', action='store_false', default=1,
628 help='Do not install optimised compiled .pyo files (configuration) [Default:install]')
629 pyopt.add_option('--nopycache',dest='nopycache', action='store_true',
630 help='Do not use __pycache__ directory to install objects [Default:auto]')
631 pyopt.add_option('--python', dest="python",
632 help='python binary to be used [Default: %s]' % sys.executable)
633 pyopt.add_option('--pythondir', dest='pythondir',
634 help='Installation path for python modules (py, platform-independent .py and .pyc files)')
635 pyopt.add_option('--pythonarchdir', dest='pythonarchdir',
636 help='Installation path for python extension (pyext, platform-dependent .so or .dylib files)')