s4-waf: updates for the new python installer from jelmer
[sfrench/samba-autobuild/.git] / buildtools / wafsamba / wafsamba.py
1 # a waf tool to add autoconf-like macros to the configure section
2 # and for SAMBA_ macros for building libraries, binaries etc
3
4 import Build, os, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs
5 from Configure import conf
6 from Logs import debug
7 from samba_utils import SUBST_VARS_RECURSIVE
8
9 # bring in the other samba modules
10 from samba_optimisation import *
11 from samba_utils import *
12 from samba_autoconf import *
13 from samba_patterns import *
14 from samba_pidl import *
15 from samba_errtable import *
16 from samba_asn1 import *
17 from samba_autoproto import *
18 from samba_python import *
19 from samba_deps import *
20 from samba_bundled import *
21 import samba_install
22 import samba_conftests
23
24 # some systems have broken threading in python
25 if os.environ.get('WAF_NOTHREADS') == '1':
26     import nothreads
27
28 LIB_PATH="shared"
29
30 os.putenv('PYTHONUNBUFFERED', '1')
31
32 @conf
33 def SAMBA_BUILD_ENV(conf):
34     '''create the samba build environment'''
35     conf.env['BUILD_DIRECTORY'] = conf.blddir
36     mkdir_p(os.path.join(conf.blddir, LIB_PATH))
37     mkdir_p(os.path.join(conf.blddir, 'python/samba/dcerpc'))
38     # this allows all of the bin/shared and bin/python targets
39     # to be expressed in terms of build directory paths
40     for p in ['python','shared']:
41         link_target = os.path.join(conf.blddir, 'default/' + p)
42         if not os.path.lexists(link_target):
43             os.symlink('../' + p, link_target)
44
45     # get perl to put the blib files in the build directory
46     blib_bld = os.path.join(conf.blddir, 'default/pidl/blib')
47     blib_src = os.path.join(conf.srcdir, 'pidl/blib')
48     mkdir_p(blib_bld + '/man1')
49     mkdir_p(blib_bld + '/man3')
50     if os.path.islink(blib_src):
51         os.unlink(blib_src)
52     elif os.path.exists(blib_src):
53         shutil.rmtree(blib_src)
54     os.symlink(blib_bld, blib_src)
55
56
57
58 def ADD_INIT_FUNCTION(bld, subsystem, target, init_function):
59     '''add an init_function to the list for a subsystem'''
60     if init_function is None:
61         return
62     bld.ASSERT(subsystem is not None, "You must specify a subsystem for init_function '%s'" % init_function)
63     cache = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
64     if not subsystem in cache:
65         cache[subsystem] = []
66     cache[subsystem].append( { 'TARGET':target, 'INIT_FUNCTION':init_function } )
67 Build.BuildContext.ADD_INIT_FUNCTION = ADD_INIT_FUNCTION
68
69
70
71 #################################################################
72 def SAMBA_LIBRARY(bld, libname, source,
73                   deps='',
74                   public_deps='',
75                   includes='',
76                   public_headers=None,
77                   header_path=None,
78                   pc_files=None,
79                   vnum=None,
80                   cflags='',
81                   external_library=False,
82                   realname=None,
83                   autoproto=None,
84                   group='main',
85                   depends_on='',
86                   local_include=True,
87                   vars=None,
88                   install_path=None,
89                   install=True,
90                   needs_python=False,
91                   target_type='LIBRARY',
92                   bundled_extension=True,
93                   link_name=None,
94                   enabled=True):
95     '''define a Samba library'''
96
97     if not enabled:
98         SET_TARGET_TYPE(bld, libname, 'DISABLED')
99         return
100
101     source = bld.EXPAND_VARIABLES(source, vars=vars)
102
103     # remember empty libraries, so we can strip the dependencies
104     if (source == '') or (source == []):
105         SET_TARGET_TYPE(bld, libname, 'EMPTY')
106         return
107
108     if target_type != 'PYTHON' and BUILTIN_LIBRARY(bld, libname):
109         obj_target = libname
110     else:
111         obj_target = libname + '.objlist'
112
113     # first create a target for building the object files for this library
114     # by separating in this way, we avoid recompiling the C files
115     # separately for the install library and the build library
116     bld.SAMBA_SUBSYSTEM(obj_target,
117                         source         = source,
118                         deps           = deps,
119                         public_deps    = public_deps,
120                         includes       = includes,
121                         public_headers = public_headers,
122                         header_path    = header_path,
123                         cflags         = cflags,
124                         group          = group,
125                         autoproto      = autoproto,
126                         depends_on     = depends_on,
127                         needs_python   = needs_python,
128                         local_include  = local_include)
129
130     if libname == obj_target:
131         return
132
133     if not SET_TARGET_TYPE(bld, libname, target_type):
134         return
135
136     # the library itself will depend on that object target
137     deps += ' ' + public_deps
138     deps = TO_LIST(deps)
139     deps.append(obj_target)
140
141     if target_type == 'PYTHON':
142         bundled_name = libname
143     else:
144         bundled_name = BUNDLED_NAME(bld, libname, bundled_extension)
145
146     features = 'cc cshlib symlink_lib install_lib'
147     if target_type == 'PYTHON':
148         features += ' pyext'
149     elif needs_python:
150         features += ' pyembed'
151
152     bld.SET_BUILD_GROUP(group)
153     t = bld(
154         features        = features,
155         source          = [],
156         target          = bundled_name,
157         samba_cflags    = CURRENT_CFLAGS(bld, libname, cflags),
158         depends_on      = depends_on,
159         samba_deps      = deps,
160         samba_includes  = includes,
161         local_include   = local_include,
162         vnum            = vnum,
163         install_path    = None,
164         samba_inst_path = install_path,
165         name            = libname,
166         samba_realname  = realname,
167         samba_install   = install
168         )
169
170     if link_name:
171         t.link_name = link_name
172
173     if autoproto is not None:
174         bld.SAMBA_AUTOPROTO(autoproto, source)
175
176     if public_headers is not None:
177         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
178
179     if pc_files is not None:
180         bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
181
182 Build.BuildContext.SAMBA_LIBRARY = SAMBA_LIBRARY
183
184
185 #################################################################
186 def SAMBA_BINARY(bld, binname, source,
187                  deps='',
188                  includes='',
189                  public_headers=None,
190                  header_path=None,
191                  modules=None,
192                  installdir=None,
193                  ldflags=None,
194                  cflags='',
195                  autoproto=None,
196                  use_hostcc=None,
197                  compiler=None,
198                  group='binaries',
199                  manpages=None,
200                  local_include=True,
201                  subsystem_name=None,
202                  needs_python=False,
203                  vars=None,
204                  install=True,
205                  install_path=None):
206     '''define a Samba binary'''
207
208     if not SET_TARGET_TYPE(bld, binname, 'BINARY'):
209         return
210
211     features = 'cc cprogram symlink_bin install_bin'
212     if needs_python:
213         features += ' pyembed'
214
215     obj_target = binname + '.objlist'
216
217     source = bld.EXPAND_VARIABLES(source, vars=vars)
218
219     # first create a target for building the object files for this binary
220     # by separating in this way, we avoid recompiling the C files
221     # separately for the install binary and the build binary
222     bld.SAMBA_SUBSYSTEM(obj_target,
223                         source         = source,
224                         deps           = deps,
225                         includes       = includes,
226                         cflags         = cflags,
227                         group          = group,
228                         autoproto      = autoproto,
229                         subsystem_name = subsystem_name,
230                         needs_python   = needs_python,
231                         local_include  = local_include)
232
233     bld.SET_BUILD_GROUP(group)
234
235     # the binary itself will depend on that object target
236     deps = TO_LIST(deps)
237     deps.append(obj_target)
238
239     t = bld(
240         features       = features,
241         source         = [],
242         target         = binname,
243         samba_cflags   = CURRENT_CFLAGS(bld, binname, cflags),
244         samba_deps     = deps,
245         samba_includes = includes,
246         local_include  = local_include,
247         samba_modules  = modules,
248         top            = True,
249         samba_subsystem= subsystem_name,
250         install_path   = None,
251         samba_inst_path= install_path,
252         samba_install  = install
253         )
254
255     # setup the subsystem_name as an alias for the real
256     # binary name, so it can be found when expanding
257     # subsystem dependencies
258     if subsystem_name is not None:
259         bld.TARGET_ALIAS(subsystem_name, binname)
260
261     if autoproto is not None:
262         bld.SAMBA_AUTOPROTO(autoproto, source)
263     if public_headers is not None:
264         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
265 Build.BuildContext.SAMBA_BINARY = SAMBA_BINARY
266
267
268 #################################################################
269 def SAMBA_MODULE(bld, modname, source,
270                  deps='',
271                  includes='',
272                  subsystem=None,
273                  init_function=None,
274                  autoproto=None,
275                  autoproto_extra_source='',
276                  aliases=None,
277                  cflags='',
278                  internal_module=True,
279                  local_include=True,
280                  vars=None,
281                  enabled=True):
282     '''define a Samba module.'''
283
284     # we add the init function regardless of whether the module
285     # is enabled or not, as we need to generate a null list if
286     # all disabled
287     bld.ADD_INIT_FUNCTION(subsystem, modname, init_function)
288
289     if internal_module or BUILTIN_LIBRARY(bld, modname):
290         # treat internal modules as subsystems for now
291         SAMBA_SUBSYSTEM(bld, modname, source,
292                         deps=deps,
293                         includes=includes,
294                         autoproto=autoproto,
295                         autoproto_extra_source=autoproto_extra_source,
296                         cflags=cflags,
297                         local_include=local_include,
298                         enabled=enabled)
299         return
300
301     if not enabled:
302         SET_TARGET_TYPE(bld, modname, 'DISABLED')
303         return
304
305     source = bld.EXPAND_VARIABLES(source, vars=vars)
306
307     # remember empty modules, so we can strip the dependencies
308     if (source == '') or (source == []):
309         SET_TARGET_TYPE(bld, modname, 'EMPTY')
310         return
311
312     if not SET_TARGET_TYPE(bld, modname, 'MODULE'):
313         return
314
315     if subsystem is not None:
316         deps += ' ' + subsystem
317
318     bld.SET_BUILD_GROUP('main')
319     bld(
320         features       = 'cc',
321         source         = source,
322         target         = modname,
323         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
324         samba_includes = includes,
325         local_include  = local_include,
326         samba_deps     = TO_LIST(deps)
327         )
328
329     if autoproto is not None:
330         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
331
332 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
333
334
335 #################################################################
336 def SAMBA_SUBSYSTEM(bld, modname, source,
337                     deps='',
338                     public_deps='',
339                     includes='',
340                     public_headers=None,
341                     header_path=None,
342                     cflags='',
343                     cflags_end=None,
344                     group='main',
345                     init_function_sentinal=None,
346                     heimdal_autoproto=None,
347                     heimdal_autoproto_options=None,
348                     heimdal_autoproto_private=None,
349                     autoproto=None,
350                     autoproto_extra_source='',
351                     depends_on='',
352                     local_include=True,
353                     local_include_first=True,
354                     subsystem_name=None,
355                     enabled=True,
356                     vars=None,
357                     needs_python=False):
358     '''define a Samba subsystem'''
359
360     if not enabled:
361         SET_TARGET_TYPE(bld, modname, 'DISABLED')
362         return
363
364     # remember empty subsystems, so we can strip the dependencies
365     if (source == '') or (source == []):
366         SET_TARGET_TYPE(bld, modname, 'EMPTY')
367         return
368
369     if not SET_TARGET_TYPE(bld, modname, 'SUBSYSTEM'):
370         return
371
372     source = bld.EXPAND_VARIABLES(source, vars=vars)
373
374     deps += ' ' + public_deps
375
376     bld.SET_BUILD_GROUP(group)
377
378     features = 'cc'
379     if needs_python:
380         features += ' pyext'
381
382     t = bld(
383         features       = features,
384         source         = source,
385         target         = modname,
386         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
387         depends_on     = depends_on,
388         samba_deps     = TO_LIST(deps),
389         samba_includes = includes,
390         local_include  = local_include,
391         local_include_first  = local_include_first,
392         samba_subsystem= subsystem_name
393         )
394
395     if cflags_end is not None:
396         t.samba_cflags.extend(TO_LIST(cflags_end))
397
398     if heimdal_autoproto is not None:
399         bld.HEIMDAL_AUTOPROTO(heimdal_autoproto, source, options=heimdal_autoproto_options)
400     if heimdal_autoproto_private is not None:
401         bld.HEIMDAL_AUTOPROTO_PRIVATE(heimdal_autoproto_private, source)
402     if autoproto is not None:
403         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
404     if public_headers is not None:
405         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
406     return t
407
408
409 Build.BuildContext.SAMBA_SUBSYSTEM = SAMBA_SUBSYSTEM
410
411
412 def SAMBA_GENERATOR(bld, name, rule, source, target,
413                     group='generators', enabled=True,
414                     public_headers=None,
415                     header_path=None,
416                     vars=None):
417     '''A generic source generator target'''
418
419     if not SET_TARGET_TYPE(bld, name, 'GENERATOR'):
420         return
421
422     if not enabled:
423         return
424
425     bld.SET_BUILD_GROUP(group)
426     t = bld(
427         rule=rule,
428         source=bld.EXPAND_VARIABLES(source, vars=vars),
429         target=target,
430         shell=isinstance(rule, str),
431         on_results=True,
432         before='cc',
433         ext_out='.c',
434         name=name)
435
436     if public_headers is not None:
437         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
438     return t
439 Build.BuildContext.SAMBA_GENERATOR = SAMBA_GENERATOR
440
441
442
443 def BUILD_SUBDIR(bld, dir):
444     '''add a new set of build rules from a subdirectory'''
445     path = os.path.normpath(bld.curdir + '/' + dir)
446     cache = LOCAL_CACHE(bld, 'SUBDIR_LIST')
447     if path in cache: return
448     cache[path] = True
449     debug("build: Processing subdirectory %s" % dir)
450     bld.add_subdirs(dir)
451 Build.BuildContext.BUILD_SUBDIR = BUILD_SUBDIR
452
453
454
455 @runonce
456 def SETUP_BUILD_GROUPS(bld):
457     '''setup build groups used to ensure that the different build
458     phases happen consecutively'''
459     bld.p_ln = bld.srcnode # we do want to see all targets!
460     bld.env['USING_BUILD_GROUPS'] = True
461     bld.add_group('setup')
462     bld.add_group('build_compiler_source')
463     bld.add_group('base_libraries')
464     bld.add_group('generators')
465     bld.add_group('compiler_prototypes')
466     bld.add_group('compiler_libraries')
467     bld.add_group('build_compilers')
468     bld.add_group('build_source')
469     bld.add_group('prototypes')
470     bld.add_group('main')
471     bld.add_group('binaries')
472     bld.add_group('final')
473 Build.BuildContext.SETUP_BUILD_GROUPS = SETUP_BUILD_GROUPS
474
475
476 def SET_BUILD_GROUP(bld, group):
477     '''set the current build group'''
478     if not 'USING_BUILD_GROUPS' in bld.env:
479         return
480     bld.set_group(group)
481 Build.BuildContext.SET_BUILD_GROUP = SET_BUILD_GROUP
482
483
484
485 @conf
486 def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
487     """use timestamps instead of file contents for deps
488     this currently doesn't work"""
489     def h_file(filename):
490         import stat
491         st = os.stat(filename)
492         if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
493         m = Utils.md5()
494         m.update(str(st.st_mtime))
495         m.update(str(st.st_size))
496         m.update(filename)
497         return m.digest()
498     Utils.h_file = h_file
499
500
501
502 t = Task.simple_task_type('copy_script', 'rm -f ${LINK_TARGET} && ln -s ${SRC[0].abspath(env)} ${LINK_TARGET}',
503                           shell=True, color='PINK', ext_in='.bin')
504 t.quiet = True
505
506 @feature('copy_script')
507 @before('apply_link')
508 def copy_script(self):
509     tsk = self.create_task('copy_script', self.allnodes[0])
510     tsk.env.TARGET = self.target
511
512 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
513     '''used to copy scripts from the source tree into the build directory
514        for use by selftest'''
515
516     source = bld.path.ant_glob(pattern)
517
518     bld.SET_BUILD_GROUP('build_source')
519     for s in TO_LIST(source):
520         iname = s
521         if installname != None:
522             iname = installname
523         target = os.path.join(installdir, iname)
524         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
525         mkdir_p(tgtdir)
526         t = bld(features='copy_script',
527                 source       = s,
528                 target       = target,
529                 always       = True,
530                 install_path = None)
531         t.env.LINK_TARGET = target
532
533 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
534
535
536 def install_file(bld, destdir, file, chmod=0644, flat=False,
537                  python_fixup=False, destname=None, base_name=None):
538     '''install a file'''
539     destdir = bld.EXPAND_VARIABLES(destdir)
540     if not destname:
541         destname = file
542         if flat:
543             destname = os.path.basename(destname)
544     dest = os.path.join(destdir, destname)
545     if python_fixup:
546         # fixup the python path it will use to find Samba modules
547         inst_file = file + '.inst'
548         bld.SAMBA_GENERATOR('python_%s' % destname,
549                             rule="sed 's|\(sys.path.insert.*\)bin/python\(.*\)$|\\1${PYTHONDIR}\\2|g' < ${SRC} > ${TGT}",
550                             source=file,
551                             target=inst_file)
552         file = inst_file
553     if base_name:
554         file = os.path.join(base_name, file)
555     bld.install_as(dest, file, chmod=chmod)
556
557
558 def INSTALL_FILES(bld, destdir, files, chmod=0644, flat=False,
559                   python_fixup=False, destname=None, base_name=None):
560     '''install a set of files'''
561     for f in TO_LIST(files):
562         install_file(bld, destdir, f, chmod=chmod, flat=flat,
563                      python_fixup=python_fixup, destname=destname,
564                      base_name=base_name)
565 Build.BuildContext.INSTALL_FILES = INSTALL_FILES
566
567
568 def INSTALL_WILDCARD(bld, destdir, pattern, chmod=0644, flat=False,
569                      python_fixup=False, exclude=None, trim_path=None):
570     '''install a set of files matching a wildcard pattern'''
571     files=TO_LIST(bld.path.ant_glob(pattern))
572     if trim_path:
573         files2 = []
574         for f in files:
575             files2.append(os_path_relpath(f, trim_path))
576         files = files2
577
578     if exclude:
579         for f in files[:]:
580             if fnmatch.fnmatch(f, exclude):
581                 files.remove(f)
582     INSTALL_FILES(bld, destdir, files, chmod=chmod, flat=flat,
583                   python_fixup=python_fixup, base_name=trim_path)
584 Build.BuildContext.INSTALL_WILDCARD = INSTALL_WILDCARD
585
586
587 def PUBLIC_HEADERS(bld, public_headers, header_path=None):
588     '''install some headers
589
590     header_path may either be a string that is added to the INCLUDEDIR,
591     or it can be a dictionary of wildcard patterns which map to destination
592     directories relative to INCLUDEDIR
593     '''
594     dest = '${INCLUDEDIR}'
595     if isinstance(header_path, str):
596         dest += '/' + header_path
597     for h in TO_LIST(public_headers):
598         hdest = dest
599         if isinstance(header_path, list):
600             for (p1, dir) in header_path:
601                 found_match=False
602                 lst = TO_LIST(p1)
603                 for p2 in lst:
604                     if fnmatch.fnmatch(h, p2):
605                         if dir:
606                             hdest = os.path.join(hdest, dir)
607                         found_match=True
608                         break
609                 if found_match: break
610         if h.find(':') != -1:
611             hs=h.split(':')
612             INSTALL_FILES(bld, hdest, hs[0], flat=True, destname=hs[1])
613         else:
614             INSTALL_FILES(bld, hdest, h, flat=True)
615 Build.BuildContext.PUBLIC_HEADERS = PUBLIC_HEADERS
616
617
618 def subst_at_vars(task):
619     '''substiture @VAR@ style variables in a file'''
620     src = task.inputs[0].srcpath(task.env)
621     tgt = task.outputs[0].bldpath(task.env)
622
623     f = open(src, 'r')
624     s = f.read()
625     f.close()
626     # split on the vars
627     a = re.split('(@\w+@)', s)
628     out = []
629     for v in a:
630         if re.match('@\w+@', v):
631             vname = v[1:-1]
632             if not vname in task.env and vname.upper() in task.env:
633                 vname = vname.upper()
634             if not vname in task.env:
635                 print "Unknown substitution %s in %s" % (v, task.name)
636                 raise
637             v = task.env[vname]
638         out.append(v)
639     contents = ''.join(out)
640     f = open(tgt, 'w')
641     s = f.write(contents)
642     f.close()
643     return 0
644
645
646
647 def PKG_CONFIG_FILES(bld, pc_files, vnum=None):
648     '''install some pkg_config pc files'''
649     dest = '${PKGCONFIGDIR}'
650     dest = bld.EXPAND_VARIABLES(dest)
651     for f in TO_LIST(pc_files):
652         base=os.path.basename(f)
653         t = bld.SAMBA_GENERATOR('PKGCONFIG_%s' % base,
654                                 rule=subst_at_vars,
655                                 source=f+'.in',
656                                 target=f)
657         if vnum:
658             t.env.PACKAGE_VERSION = vnum
659         INSTALL_FILES(bld, dest, f, flat=True, destname=base)
660 Build.BuildContext.PKG_CONFIG_FILES = PKG_CONFIG_FILES
661
662
663
664 #############################################################
665 # give a nicer display when building different types of files
666 def progress_display(self, msg, fname):
667     col1 = Logs.colors(self.color)
668     col2 = Logs.colors.NORMAL
669     total = self.position[1]
670     n = len(str(total))
671     fs = '[%%%dd/%%%dd] %s %%s%%s%%s\n' % (n, n, msg)
672     return fs % (self.position[0], self.position[1], col1, fname, col2)
673
674 def link_display(self):
675     if Options.options.progress_bar != 0:
676         return Task.Task.old_display(self)
677     fname = self.outputs[0].bldpath(self.env)
678     return progress_display(self, 'Linking', fname)
679 Task.TaskBase.classes['cc_link'].display = link_display
680
681 def samba_display(self):
682     if Options.options.progress_bar != 0:
683         return Task.Task.old_display(self)
684     fname = self.inputs[0].bldpath(self.env)
685     if fname[0:3] == '../':
686         fname = fname[3:]
687     ext_loc = fname.rfind('.')
688     if ext_loc == -1:
689         return Task.Task.old_display(self)
690     ext = fname[ext_loc:]
691
692     ext_map = { '.idl' : 'Compiling IDL',
693                 '.et'  : 'Compiling ERRTABLE',
694                 '.asn1': 'Compiling ASN1',
695                 '.c'   : 'Compiling' }
696     if ext in ext_map:
697         return progress_display(self, ext_map[ext], fname)
698     return Task.Task.old_display(self)
699
700 Task.TaskBase.classes['Task'].old_display = Task.TaskBase.classes['Task'].display
701 Task.TaskBase.classes['Task'].display = samba_display