s4-waf: move to a universal method of recursing into subdirs
[sfrench/samba-autobuild/.git] / buildtools / wafsamba / wafsamba.py
1 # a waf tool to add autoconf-like macros to the configure section
2 # and for SAMBA_ macros for building libraries, binaries etc
3
4 import Build, os, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs
5 from Configure import conf
6 from Logs import debug
7 from samba_utils import SUBST_VARS_RECURSIVE
8
9 # bring in the other samba modules
10 from samba_optimisation import *
11 from samba_utils import *
12 from samba_autoconf import *
13 from samba_patterns import *
14 from samba_pidl import *
15 from samba_errtable import *
16 from samba_asn1 import *
17 from samba_autoproto import *
18 from samba_python import *
19 from samba_deps import *
20 from samba_bundled import *
21 import samba_install
22 import samba_conftests
23 import tru64cc
24 import irixcc
25 import generic_cc
26 import samba_dist
27
28 # some systems have broken threading in python
29 if os.environ.get('WAF_NOTHREADS') == '1':
30     import nothreads
31
32 LIB_PATH="shared"
33
34 os.putenv('PYTHONUNBUFFERED', '1')
35
36 @conf
37 def SAMBA_BUILD_ENV(conf):
38     '''create the samba build environment'''
39     conf.env['BUILD_DIRECTORY'] = conf.blddir
40     mkdir_p(os.path.join(conf.blddir, LIB_PATH))
41     mkdir_p(os.path.join(conf.blddir, 'python/samba/dcerpc'))
42     # this allows all of the bin/shared and bin/python targets
43     # to be expressed in terms of build directory paths
44     for p in ['python','shared']:
45         link_target = os.path.join(conf.blddir, 'default/' + p)
46         if not os.path.lexists(link_target):
47             os.symlink('../' + p, link_target)
48
49     # get perl to put the blib files in the build directory
50     blib_bld = os.path.join(conf.blddir, 'default/pidl/blib')
51     blib_src = os.path.join(conf.srcdir, 'pidl/blib')
52     mkdir_p(blib_bld + '/man1')
53     mkdir_p(blib_bld + '/man3')
54     if os.path.islink(blib_src):
55         os.unlink(blib_src)
56     elif os.path.exists(blib_src):
57         shutil.rmtree(blib_src)
58
59
60 def ADD_INIT_FUNCTION(bld, subsystem, target, init_function):
61     '''add an init_function to the list for a subsystem'''
62     if init_function is None:
63         return
64     bld.ASSERT(subsystem is not None, "You must specify a subsystem for init_function '%s'" % init_function)
65     cache = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
66     if not subsystem in cache:
67         cache[subsystem] = []
68     cache[subsystem].append( { 'TARGET':target, 'INIT_FUNCTION':init_function } )
69 Build.BuildContext.ADD_INIT_FUNCTION = ADD_INIT_FUNCTION
70
71
72
73 #################################################################
74 def SAMBA_LIBRARY(bld, libname, source,
75                   deps='',
76                   public_deps='',
77                   includes='',
78                   public_headers=None,
79                   header_path=None,
80                   pc_files=None,
81                   vnum=None,
82                   cflags='',
83                   external_library=False,
84                   realname=None,
85                   autoproto=None,
86                   group='main',
87                   depends_on='',
88                   local_include=True,
89                   vars=None,
90                   install_path=None,
91                   install=True,
92                   needs_python=False,
93                   target_type='LIBRARY',
94                   bundled_extension=True,
95                   link_name=None,
96                   enabled=True):
97     '''define a Samba library'''
98
99     if not enabled:
100         SET_TARGET_TYPE(bld, libname, 'DISABLED')
101         return
102
103     source = bld.EXPAND_VARIABLES(source, vars=vars)
104
105     # remember empty libraries, so we can strip the dependencies
106     if (source == '') or (source == []):
107         SET_TARGET_TYPE(bld, libname, 'EMPTY')
108         return
109
110     if target_type != 'PYTHON' and BUILTIN_LIBRARY(bld, libname):
111         obj_target = libname
112     else:
113         obj_target = libname + '.objlist'
114
115     # first create a target for building the object files for this library
116     # by separating in this way, we avoid recompiling the C files
117     # separately for the install library and the build library
118     bld.SAMBA_SUBSYSTEM(obj_target,
119                         source         = source,
120                         deps           = deps,
121                         public_deps    = public_deps,
122                         includes       = includes,
123                         public_headers = public_headers,
124                         header_path    = header_path,
125                         cflags         = cflags,
126                         group          = group,
127                         autoproto      = autoproto,
128                         depends_on     = depends_on,
129                         needs_python   = needs_python,
130                         local_include  = local_include)
131
132     if libname == obj_target:
133         return
134
135     if not SET_TARGET_TYPE(bld, libname, target_type):
136         return
137
138     # the library itself will depend on that object target
139     deps += ' ' + public_deps
140     deps = TO_LIST(deps)
141     deps.append(obj_target)
142
143     if target_type == 'PYTHON':
144         bundled_name = libname
145     else:
146         bundled_name = BUNDLED_NAME(bld, libname, bundled_extension)
147
148     features = 'cc cshlib symlink_lib install_lib'
149     if target_type == 'PYTHON':
150         features += ' pyext'
151     elif needs_python:
152         features += ' pyembed'
153
154     bld.SET_BUILD_GROUP(group)
155     t = bld(
156         features        = features,
157         source          = [],
158         target          = bundled_name,
159         samba_cflags    = CURRENT_CFLAGS(bld, libname, cflags),
160         depends_on      = depends_on,
161         samba_deps      = deps,
162         samba_includes  = includes,
163         local_include   = local_include,
164         vnum            = vnum,
165         install_path    = None,
166         samba_inst_path = install_path,
167         name            = libname,
168         samba_realname  = realname,
169         samba_install   = install
170         )
171
172     if link_name:
173         t.link_name = link_name
174
175     if autoproto is not None:
176         bld.SAMBA_AUTOPROTO(autoproto, source)
177
178     if public_headers is not None:
179         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
180
181     if pc_files is not None:
182         bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
183
184 Build.BuildContext.SAMBA_LIBRARY = SAMBA_LIBRARY
185
186
187 #################################################################
188 def SAMBA_BINARY(bld, binname, source,
189                  deps='',
190                  includes='',
191                  public_headers=None,
192                  header_path=None,
193                  modules=None,
194                  installdir=None,
195                  ldflags=None,
196                  cflags='',
197                  autoproto=None,
198                  use_hostcc=None,
199                  compiler=None,
200                  group='binaries',
201                  manpages=None,
202                  local_include=True,
203                  subsystem_name=None,
204                  needs_python=False,
205                  vars=None,
206                  install=True,
207                  install_path=None):
208     '''define a Samba binary'''
209
210     if not SET_TARGET_TYPE(bld, binname, 'BINARY'):
211         return
212
213     features = 'cc cprogram symlink_bin install_bin'
214     if needs_python:
215         features += ' pyembed'
216
217     obj_target = binname + '.objlist'
218
219     source = bld.EXPAND_VARIABLES(source, vars=vars)
220
221     # first create a target for building the object files for this binary
222     # by separating in this way, we avoid recompiling the C files
223     # separately for the install binary and the build binary
224     bld.SAMBA_SUBSYSTEM(obj_target,
225                         source         = source,
226                         deps           = deps,
227                         includes       = includes,
228                         cflags         = cflags,
229                         group          = group,
230                         autoproto      = autoproto,
231                         subsystem_name = subsystem_name,
232                         needs_python   = needs_python,
233                         local_include  = local_include)
234
235     bld.SET_BUILD_GROUP(group)
236
237     # the binary itself will depend on that object target
238     deps = TO_LIST(deps)
239     deps.append(obj_target)
240
241     t = bld(
242         features       = features,
243         source         = [],
244         target         = binname,
245         samba_cflags   = CURRENT_CFLAGS(bld, binname, cflags),
246         samba_deps     = deps,
247         samba_includes = includes,
248         local_include  = local_include,
249         samba_modules  = modules,
250         top            = True,
251         samba_subsystem= subsystem_name,
252         install_path   = None,
253         samba_inst_path= install_path,
254         samba_install  = install
255         )
256
257     # setup the subsystem_name as an alias for the real
258     # binary name, so it can be found when expanding
259     # subsystem dependencies
260     if subsystem_name is not None:
261         bld.TARGET_ALIAS(subsystem_name, binname)
262
263     if autoproto is not None:
264         bld.SAMBA_AUTOPROTO(autoproto, source)
265     if public_headers is not None:
266         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
267 Build.BuildContext.SAMBA_BINARY = SAMBA_BINARY
268
269
270 #################################################################
271 def SAMBA_MODULE(bld, modname, source,
272                  deps='',
273                  includes='',
274                  subsystem=None,
275                  init_function=None,
276                  autoproto=None,
277                  autoproto_extra_source='',
278                  aliases=None,
279                  cflags='',
280                  internal_module=True,
281                  local_include=True,
282                  vars=None,
283                  enabled=True):
284     '''define a Samba module.'''
285
286     # we add the init function regardless of whether the module
287     # is enabled or not, as we need to generate a null list if
288     # all disabled
289     bld.ADD_INIT_FUNCTION(subsystem, modname, init_function)
290
291     if internal_module or BUILTIN_LIBRARY(bld, modname):
292         # treat internal modules as subsystems for now
293         SAMBA_SUBSYSTEM(bld, modname, source,
294                         deps=deps,
295                         includes=includes,
296                         autoproto=autoproto,
297                         autoproto_extra_source=autoproto_extra_source,
298                         cflags=cflags,
299                         local_include=local_include,
300                         enabled=enabled)
301         return
302
303     if not enabled:
304         SET_TARGET_TYPE(bld, modname, 'DISABLED')
305         return
306
307     source = bld.EXPAND_VARIABLES(source, vars=vars)
308
309     # remember empty modules, so we can strip the dependencies
310     if (source == '') or (source == []):
311         SET_TARGET_TYPE(bld, modname, 'EMPTY')
312         return
313
314     if not SET_TARGET_TYPE(bld, modname, 'MODULE'):
315         return
316
317     if subsystem is not None:
318         deps += ' ' + subsystem
319
320     bld.SET_BUILD_GROUP('main')
321     bld(
322         features       = 'cc',
323         source         = source,
324         target         = modname,
325         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
326         samba_includes = includes,
327         local_include  = local_include,
328         samba_deps     = TO_LIST(deps)
329         )
330
331     if autoproto is not None:
332         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
333
334 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
335
336
337 #################################################################
338 def SAMBA_SUBSYSTEM(bld, modname, source,
339                     deps='',
340                     public_deps='',
341                     includes='',
342                     public_headers=None,
343                     header_path=None,
344                     cflags='',
345                     cflags_end=None,
346                     group='main',
347                     init_function_sentinal=None,
348                     heimdal_autoproto=None,
349                     heimdal_autoproto_options=None,
350                     heimdal_autoproto_private=None,
351                     autoproto=None,
352                     autoproto_extra_source='',
353                     depends_on='',
354                     local_include=True,
355                     local_include_first=True,
356                     subsystem_name=None,
357                     enabled=True,
358                     vars=None,
359                     needs_python=False):
360     '''define a Samba subsystem'''
361
362     if not enabled:
363         SET_TARGET_TYPE(bld, modname, 'DISABLED')
364         return
365
366     # remember empty subsystems, so we can strip the dependencies
367     if (source == '') or (source == []):
368         SET_TARGET_TYPE(bld, modname, 'EMPTY')
369         return
370
371     if not SET_TARGET_TYPE(bld, modname, 'SUBSYSTEM'):
372         return
373
374     source = bld.EXPAND_VARIABLES(source, vars=vars)
375
376     deps += ' ' + public_deps
377
378     bld.SET_BUILD_GROUP(group)
379
380     features = 'cc'
381     if needs_python:
382         features += ' pyext'
383
384     t = bld(
385         features       = features,
386         source         = source,
387         target         = modname,
388         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
389         depends_on     = depends_on,
390         samba_deps     = TO_LIST(deps),
391         samba_includes = includes,
392         local_include  = local_include,
393         local_include_first  = local_include_first,
394         samba_subsystem= subsystem_name
395         )
396
397     if cflags_end is not None:
398         t.samba_cflags.extend(TO_LIST(cflags_end))
399
400     if heimdal_autoproto is not None:
401         bld.HEIMDAL_AUTOPROTO(heimdal_autoproto, source, options=heimdal_autoproto_options)
402     if heimdal_autoproto_private is not None:
403         bld.HEIMDAL_AUTOPROTO_PRIVATE(heimdal_autoproto_private, source)
404     if autoproto is not None:
405         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
406     if public_headers is not None:
407         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
408     return t
409
410
411 Build.BuildContext.SAMBA_SUBSYSTEM = SAMBA_SUBSYSTEM
412
413
414 def SAMBA_GENERATOR(bld, name, rule, source, target,
415                     group='generators', enabled=True,
416                     public_headers=None,
417                     header_path=None,
418                     vars=None):
419     '''A generic source generator target'''
420
421     if not SET_TARGET_TYPE(bld, name, 'GENERATOR'):
422         return
423
424     if not enabled:
425         return
426
427     bld.SET_BUILD_GROUP(group)
428     t = bld(
429         rule=rule,
430         source=bld.EXPAND_VARIABLES(source, vars=vars),
431         target=target,
432         shell=isinstance(rule, str),
433         on_results=True,
434         before='cc',
435         ext_out='.c',
436         name=name)
437
438     if public_headers is not None:
439         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
440     return t
441 Build.BuildContext.SAMBA_GENERATOR = SAMBA_GENERATOR
442
443
444
445 @runonce
446 def SETUP_BUILD_GROUPS(bld):
447     '''setup build groups used to ensure that the different build
448     phases happen consecutively'''
449     bld.p_ln = bld.srcnode # we do want to see all targets!
450     bld.env['USING_BUILD_GROUPS'] = True
451     bld.add_group('setup')
452     bld.add_group('build_compiler_source')
453     bld.add_group('base_libraries')
454     bld.add_group('generators')
455     bld.add_group('compiler_prototypes')
456     bld.add_group('compiler_libraries')
457     bld.add_group('build_compilers')
458     bld.add_group('build_source')
459     bld.add_group('prototypes')
460     bld.add_group('main')
461     bld.add_group('binaries')
462     bld.add_group('final')
463 Build.BuildContext.SETUP_BUILD_GROUPS = SETUP_BUILD_GROUPS
464
465
466 def SET_BUILD_GROUP(bld, group):
467     '''set the current build group'''
468     if not 'USING_BUILD_GROUPS' in bld.env:
469         return
470     bld.set_group(group)
471 Build.BuildContext.SET_BUILD_GROUP = SET_BUILD_GROUP
472
473
474
475 @conf
476 def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
477     """use timestamps instead of file contents for deps
478     this currently doesn't work"""
479     def h_file(filename):
480         import stat
481         st = os.stat(filename)
482         if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
483         m = Utils.md5()
484         m.update(str(st.st_mtime))
485         m.update(str(st.st_size))
486         m.update(filename)
487         return m.digest()
488     Utils.h_file = h_file
489
490
491
492 t = Task.simple_task_type('copy_script', 'rm -f ${LINK_TARGET} && ln -s ${SRC[0].abspath(env)} ${LINK_TARGET}',
493                           shell=True, color='PINK', ext_in='.bin')
494 t.quiet = True
495
496 @feature('copy_script')
497 @before('apply_link')
498 def copy_script(self):
499     tsk = self.create_task('copy_script', self.allnodes[0])
500     tsk.env.TARGET = self.target
501
502 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
503     '''used to copy scripts from the source tree into the build directory
504        for use by selftest'''
505
506     source = bld.path.ant_glob(pattern)
507
508     bld.SET_BUILD_GROUP('build_source')
509     for s in TO_LIST(source):
510         iname = s
511         if installname != None:
512             iname = installname
513         target = os.path.join(installdir, iname)
514         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
515         mkdir_p(tgtdir)
516         t = bld(features='copy_script',
517                 source       = s,
518                 target       = target,
519                 always       = True,
520                 install_path = None)
521         t.env.LINK_TARGET = target
522
523 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
524
525
526 def install_file(bld, destdir, file, chmod=0644, flat=False,
527                  python_fixup=False, destname=None, base_name=None):
528     '''install a file'''
529     destdir = bld.EXPAND_VARIABLES(destdir)
530     if not destname:
531         destname = file
532         if flat:
533             destname = os.path.basename(destname)
534     dest = os.path.join(destdir, destname)
535     if python_fixup:
536         # fixup the python path it will use to find Samba modules
537         inst_file = file + '.inst'
538         bld.SAMBA_GENERATOR('python_%s' % destname,
539                             rule="sed 's|\(sys.path.insert.*\)bin/python\(.*\)$|\\1${PYTHONDIR}\\2|g' < ${SRC} > ${TGT}",
540                             source=file,
541                             target=inst_file)
542         file = inst_file
543     if base_name:
544         file = os.path.join(base_name, file)
545     bld.install_as(dest, file, chmod=chmod)
546
547
548 def INSTALL_FILES(bld, destdir, files, chmod=0644, flat=False,
549                   python_fixup=False, destname=None, base_name=None):
550     '''install a set of files'''
551     for f in TO_LIST(files):
552         install_file(bld, destdir, f, chmod=chmod, flat=flat,
553                      python_fixup=python_fixup, destname=destname,
554                      base_name=base_name)
555 Build.BuildContext.INSTALL_FILES = INSTALL_FILES
556
557
558 def INSTALL_WILDCARD(bld, destdir, pattern, chmod=0644, flat=False,
559                      python_fixup=False, exclude=None, trim_path=None):
560     '''install a set of files matching a wildcard pattern'''
561     files=TO_LIST(bld.path.ant_glob(pattern))
562     if trim_path:
563         files2 = []
564         for f in files:
565             files2.append(os_path_relpath(f, trim_path))
566         files = files2
567
568     if exclude:
569         for f in files[:]:
570             if fnmatch.fnmatch(f, exclude):
571                 files.remove(f)
572     INSTALL_FILES(bld, destdir, files, chmod=chmod, flat=flat,
573                   python_fixup=python_fixup, base_name=trim_path)
574 Build.BuildContext.INSTALL_WILDCARD = INSTALL_WILDCARD
575
576
577 def PUBLIC_HEADERS(bld, public_headers, header_path=None):
578     '''install some headers
579
580     header_path may either be a string that is added to the INCLUDEDIR,
581     or it can be a dictionary of wildcard patterns which map to destination
582     directories relative to INCLUDEDIR
583     '''
584     dest = '${INCLUDEDIR}'
585     if isinstance(header_path, str):
586         dest += '/' + header_path
587     for h in TO_LIST(public_headers):
588         hdest = dest
589         if isinstance(header_path, list):
590             for (p1, dir) in header_path:
591                 found_match=False
592                 lst = TO_LIST(p1)
593                 for p2 in lst:
594                     if fnmatch.fnmatch(h, p2):
595                         if dir:
596                             hdest = os.path.join(hdest, dir)
597                         found_match=True
598                         break
599                 if found_match: break
600         if h.find(':') != -1:
601             hs=h.split(':')
602             INSTALL_FILES(bld, hdest, hs[0], flat=True, destname=hs[1])
603         else:
604             INSTALL_FILES(bld, hdest, h, flat=True)
605 Build.BuildContext.PUBLIC_HEADERS = PUBLIC_HEADERS
606
607
608 def subst_at_vars(task):
609     '''substiture @VAR@ style variables in a file'''
610     src = task.inputs[0].srcpath(task.env)
611     tgt = task.outputs[0].bldpath(task.env)
612
613     f = open(src, 'r')
614     s = f.read()
615     f.close()
616     # split on the vars
617     a = re.split('(@\w+@)', s)
618     out = []
619     back_sub = [ ('PREFIX', '${prefix}'), ('EXEC_PREFIX', '${exec_prefix}')]
620     for v in a:
621         if re.match('@\w+@', v):
622             vname = v[1:-1]
623             if not vname in task.env and vname.upper() in task.env:
624                 vname = vname.upper()
625             if not vname in task.env:
626                 print "Unknown substitution %s in %s" % (v, task.name)
627                 raise
628             v = SUBST_VARS_RECURSIVE(task.env[vname], task.env)
629             # now we back substitute the allowed pc vars
630             for (b, m) in back_sub:
631                 s = task.env[b]
632                 if s == v[0:len(s)]:
633                     v = m + v[len(s):]
634         out.append(v)
635     contents = ''.join(out)
636     f = open(tgt, 'w')
637     s = f.write(contents)
638     f.close()
639     return 0
640
641
642
643 def PKG_CONFIG_FILES(bld, pc_files, vnum=None):
644     '''install some pkg_config pc files'''
645     dest = '${PKGCONFIGDIR}'
646     dest = bld.EXPAND_VARIABLES(dest)
647     for f in TO_LIST(pc_files):
648         base=os.path.basename(f)
649         t = bld.SAMBA_GENERATOR('PKGCONFIG_%s' % base,
650                                 rule=subst_at_vars,
651                                 source=f+'.in',
652                                 target=f)
653         if vnum:
654             t.env.PACKAGE_VERSION = vnum
655         INSTALL_FILES(bld, dest, f, flat=True, destname=base)
656 Build.BuildContext.PKG_CONFIG_FILES = PKG_CONFIG_FILES
657
658
659
660 #############################################################
661 # give a nicer display when building different types of files
662 def progress_display(self, msg, fname):
663     col1 = Logs.colors(self.color)
664     col2 = Logs.colors.NORMAL
665     total = self.position[1]
666     n = len(str(total))
667     fs = '[%%%dd/%%%dd] %s %%s%%s%%s\n' % (n, n, msg)
668     return fs % (self.position[0], self.position[1], col1, fname, col2)
669
670 def link_display(self):
671     if Options.options.progress_bar != 0:
672         return Task.Task.old_display(self)
673     fname = self.outputs[0].bldpath(self.env)
674     return progress_display(self, 'Linking', fname)
675 Task.TaskBase.classes['cc_link'].display = link_display
676
677 def samba_display(self):
678     if Options.options.progress_bar != 0:
679         return Task.Task.old_display(self)
680
681     targets    = LOCAL_CACHE(self, 'TARGET_TYPE')
682     if self.name in targets:
683         target_type = targets[self.name]
684         type_map = { 'GENERATOR' : 'Generating',
685                      'PROTOTYPE' : 'Generating'
686                      }
687         if target_type in type_map:
688             return progress_display(self, type_map[target_type], self.name)
689
690     fname = self.inputs[0].bldpath(self.env)
691     if fname[0:3] == '../':
692         fname = fname[3:]
693     ext_loc = fname.rfind('.')
694     if ext_loc == -1:
695         return Task.Task.old_display(self)
696     ext = fname[ext_loc:]
697
698     ext_map = { '.idl' : 'Compiling IDL',
699                 '.et'  : 'Compiling ERRTABLE',
700                 '.asn1': 'Compiling ASN1',
701                 '.c'   : 'Compiling' }
702     if ext in ext_map:
703         return progress_display(self, ext_map[ext], fname)
704     return Task.Task.old_display(self)
705
706 Task.TaskBase.classes['Task'].old_display = Task.TaskBase.classes['Task'].display
707 Task.TaskBase.classes['Task'].display = samba_display