pidl-waf: better handling of the man page generation
[samba.git] / buildtools / wafsamba / wafsamba.py
1 # a waf tool to add autoconf-like macros to the configure section
2 # and for SAMBA_ macros for building libraries, binaries etc
3
4 import Build, os, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs
5 from Configure import conf
6 from Logs import debug
7 from samba_utils import SUBST_VARS_RECURSIVE
8
9 # bring in the other samba modules
10 from samba_optimisation import *
11 from samba_utils import *
12 from samba_autoconf import *
13 from samba_patterns import *
14 from samba_pidl import *
15 from samba_errtable import *
16 from samba_asn1 import *
17 from samba_autoproto import *
18 from samba_python import *
19 from samba_deps import *
20 from samba_bundled import *
21 import samba_install
22 import samba_conftests
23
24 # some systems have broken threading in python
25 if os.environ.get('WAF_NOTHREADS') == '1':
26     import nothreads
27
28 LIB_PATH="shared"
29
30 os.putenv('PYTHONUNBUFFERED', '1')
31
32 @conf
33 def SAMBA_BUILD_ENV(conf):
34     '''create the samba build environment'''
35     conf.env['BUILD_DIRECTORY'] = conf.blddir
36     mkdir_p(os.path.join(conf.blddir, LIB_PATH))
37     mkdir_p(os.path.join(conf.blddir, 'python/samba/dcerpc'))
38     # this allows all of the bin/shared and bin/python targets
39     # to be expressed in terms of build directory paths
40     for p in ['python','shared']:
41         link_target = os.path.join(conf.blddir, 'default/' + p)
42         if not os.path.lexists(link_target):
43             os.symlink('../' + p, link_target)
44
45     # get perl to put the blib files in the build directory
46     blib_bld = os.path.join(conf.blddir, 'default/pidl/blib')
47     blib_src = os.path.join(conf.srcdir, 'pidl/blib')
48     mkdir_p(blib_bld + '/man1')
49     mkdir_p(blib_bld + '/man3')
50     if os.path.islink(blib_src):
51         os.unlink(blib_src)
52     elif os.path.exists(blib_src):
53         shutil.rmtree(blib_src)
54
55
56 def ADD_INIT_FUNCTION(bld, subsystem, target, init_function):
57     '''add an init_function to the list for a subsystem'''
58     if init_function is None:
59         return
60     bld.ASSERT(subsystem is not None, "You must specify a subsystem for init_function '%s'" % init_function)
61     cache = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
62     if not subsystem in cache:
63         cache[subsystem] = []
64     cache[subsystem].append( { 'TARGET':target, 'INIT_FUNCTION':init_function } )
65 Build.BuildContext.ADD_INIT_FUNCTION = ADD_INIT_FUNCTION
66
67
68
69 #################################################################
70 def SAMBA_LIBRARY(bld, libname, source,
71                   deps='',
72                   public_deps='',
73                   includes='',
74                   public_headers=None,
75                   header_path=None,
76                   pc_files=None,
77                   vnum=None,
78                   cflags='',
79                   external_library=False,
80                   realname=None,
81                   autoproto=None,
82                   group='main',
83                   depends_on='',
84                   local_include=True,
85                   vars=None,
86                   install_path=None,
87                   install=True,
88                   needs_python=False,
89                   target_type='LIBRARY',
90                   bundled_extension=True,
91                   link_name=None,
92                   enabled=True):
93     '''define a Samba library'''
94
95     if not enabled:
96         SET_TARGET_TYPE(bld, libname, 'DISABLED')
97         return
98
99     source = bld.EXPAND_VARIABLES(source, vars=vars)
100
101     # remember empty libraries, so we can strip the dependencies
102     if (source == '') or (source == []):
103         SET_TARGET_TYPE(bld, libname, 'EMPTY')
104         return
105
106     if target_type != 'PYTHON' and BUILTIN_LIBRARY(bld, libname):
107         obj_target = libname
108     else:
109         obj_target = libname + '.objlist'
110
111     # first create a target for building the object files for this library
112     # by separating in this way, we avoid recompiling the C files
113     # separately for the install library and the build library
114     bld.SAMBA_SUBSYSTEM(obj_target,
115                         source         = source,
116                         deps           = deps,
117                         public_deps    = public_deps,
118                         includes       = includes,
119                         public_headers = public_headers,
120                         header_path    = header_path,
121                         cflags         = cflags,
122                         group          = group,
123                         autoproto      = autoproto,
124                         depends_on     = depends_on,
125                         needs_python   = needs_python,
126                         local_include  = local_include)
127
128     if libname == obj_target:
129         return
130
131     if not SET_TARGET_TYPE(bld, libname, target_type):
132         return
133
134     # the library itself will depend on that object target
135     deps += ' ' + public_deps
136     deps = TO_LIST(deps)
137     deps.append(obj_target)
138
139     if target_type == 'PYTHON':
140         bundled_name = libname
141     else:
142         bundled_name = BUNDLED_NAME(bld, libname, bundled_extension)
143
144     features = 'cc cshlib symlink_lib install_lib'
145     if target_type == 'PYTHON':
146         features += ' pyext'
147     elif needs_python:
148         features += ' pyembed'
149
150     bld.SET_BUILD_GROUP(group)
151     t = bld(
152         features        = features,
153         source          = [],
154         target          = bundled_name,
155         samba_cflags    = CURRENT_CFLAGS(bld, libname, cflags),
156         depends_on      = depends_on,
157         samba_deps      = deps,
158         samba_includes  = includes,
159         local_include   = local_include,
160         vnum            = vnum,
161         install_path    = None,
162         samba_inst_path = install_path,
163         name            = libname,
164         samba_realname  = realname,
165         samba_install   = install
166         )
167
168     if link_name:
169         t.link_name = link_name
170
171     if autoproto is not None:
172         bld.SAMBA_AUTOPROTO(autoproto, source)
173
174     if public_headers is not None:
175         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
176
177     if pc_files is not None:
178         bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
179
180 Build.BuildContext.SAMBA_LIBRARY = SAMBA_LIBRARY
181
182
183 #################################################################
184 def SAMBA_BINARY(bld, binname, source,
185                  deps='',
186                  includes='',
187                  public_headers=None,
188                  header_path=None,
189                  modules=None,
190                  installdir=None,
191                  ldflags=None,
192                  cflags='',
193                  autoproto=None,
194                  use_hostcc=None,
195                  compiler=None,
196                  group='binaries',
197                  manpages=None,
198                  local_include=True,
199                  subsystem_name=None,
200                  needs_python=False,
201                  vars=None,
202                  install=True,
203                  install_path=None):
204     '''define a Samba binary'''
205
206     if not SET_TARGET_TYPE(bld, binname, 'BINARY'):
207         return
208
209     features = 'cc cprogram symlink_bin install_bin'
210     if needs_python:
211         features += ' pyembed'
212
213     obj_target = binname + '.objlist'
214
215     source = bld.EXPAND_VARIABLES(source, vars=vars)
216
217     # first create a target for building the object files for this binary
218     # by separating in this way, we avoid recompiling the C files
219     # separately for the install binary and the build binary
220     bld.SAMBA_SUBSYSTEM(obj_target,
221                         source         = source,
222                         deps           = deps,
223                         includes       = includes,
224                         cflags         = cflags,
225                         group          = group,
226                         autoproto      = autoproto,
227                         subsystem_name = subsystem_name,
228                         needs_python   = needs_python,
229                         local_include  = local_include)
230
231     bld.SET_BUILD_GROUP(group)
232
233     # the binary itself will depend on that object target
234     deps = TO_LIST(deps)
235     deps.append(obj_target)
236
237     t = bld(
238         features       = features,
239         source         = [],
240         target         = binname,
241         samba_cflags   = CURRENT_CFLAGS(bld, binname, cflags),
242         samba_deps     = deps,
243         samba_includes = includes,
244         local_include  = local_include,
245         samba_modules  = modules,
246         top            = True,
247         samba_subsystem= subsystem_name,
248         install_path   = None,
249         samba_inst_path= install_path,
250         samba_install  = install
251         )
252
253     # setup the subsystem_name as an alias for the real
254     # binary name, so it can be found when expanding
255     # subsystem dependencies
256     if subsystem_name is not None:
257         bld.TARGET_ALIAS(subsystem_name, binname)
258
259     if autoproto is not None:
260         bld.SAMBA_AUTOPROTO(autoproto, source)
261     if public_headers is not None:
262         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
263 Build.BuildContext.SAMBA_BINARY = SAMBA_BINARY
264
265
266 #################################################################
267 def SAMBA_MODULE(bld, modname, source,
268                  deps='',
269                  includes='',
270                  subsystem=None,
271                  init_function=None,
272                  autoproto=None,
273                  autoproto_extra_source='',
274                  aliases=None,
275                  cflags='',
276                  internal_module=True,
277                  local_include=True,
278                  vars=None,
279                  enabled=True):
280     '''define a Samba module.'''
281
282     # we add the init function regardless of whether the module
283     # is enabled or not, as we need to generate a null list if
284     # all disabled
285     bld.ADD_INIT_FUNCTION(subsystem, modname, init_function)
286
287     if internal_module or BUILTIN_LIBRARY(bld, modname):
288         # treat internal modules as subsystems for now
289         SAMBA_SUBSYSTEM(bld, modname, source,
290                         deps=deps,
291                         includes=includes,
292                         autoproto=autoproto,
293                         autoproto_extra_source=autoproto_extra_source,
294                         cflags=cflags,
295                         local_include=local_include,
296                         enabled=enabled)
297         return
298
299     if not enabled:
300         SET_TARGET_TYPE(bld, modname, 'DISABLED')
301         return
302
303     source = bld.EXPAND_VARIABLES(source, vars=vars)
304
305     # remember empty modules, so we can strip the dependencies
306     if (source == '') or (source == []):
307         SET_TARGET_TYPE(bld, modname, 'EMPTY')
308         return
309
310     if not SET_TARGET_TYPE(bld, modname, 'MODULE'):
311         return
312
313     if subsystem is not None:
314         deps += ' ' + subsystem
315
316     bld.SET_BUILD_GROUP('main')
317     bld(
318         features       = 'cc',
319         source         = source,
320         target         = modname,
321         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
322         samba_includes = includes,
323         local_include  = local_include,
324         samba_deps     = TO_LIST(deps)
325         )
326
327     if autoproto is not None:
328         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
329
330 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
331
332
333 #################################################################
334 def SAMBA_SUBSYSTEM(bld, modname, source,
335                     deps='',
336                     public_deps='',
337                     includes='',
338                     public_headers=None,
339                     header_path=None,
340                     cflags='',
341                     cflags_end=None,
342                     group='main',
343                     init_function_sentinal=None,
344                     heimdal_autoproto=None,
345                     heimdal_autoproto_options=None,
346                     heimdal_autoproto_private=None,
347                     autoproto=None,
348                     autoproto_extra_source='',
349                     depends_on='',
350                     local_include=True,
351                     local_include_first=True,
352                     subsystem_name=None,
353                     enabled=True,
354                     vars=None,
355                     needs_python=False):
356     '''define a Samba subsystem'''
357
358     if not enabled:
359         SET_TARGET_TYPE(bld, modname, 'DISABLED')
360         return
361
362     # remember empty subsystems, so we can strip the dependencies
363     if (source == '') or (source == []):
364         SET_TARGET_TYPE(bld, modname, 'EMPTY')
365         return
366
367     if not SET_TARGET_TYPE(bld, modname, 'SUBSYSTEM'):
368         return
369
370     source = bld.EXPAND_VARIABLES(source, vars=vars)
371
372     deps += ' ' + public_deps
373
374     bld.SET_BUILD_GROUP(group)
375
376     features = 'cc'
377     if needs_python:
378         features += ' pyext'
379
380     t = bld(
381         features       = features,
382         source         = source,
383         target         = modname,
384         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
385         depends_on     = depends_on,
386         samba_deps     = TO_LIST(deps),
387         samba_includes = includes,
388         local_include  = local_include,
389         local_include_first  = local_include_first,
390         samba_subsystem= subsystem_name
391         )
392
393     if cflags_end is not None:
394         t.samba_cflags.extend(TO_LIST(cflags_end))
395
396     if heimdal_autoproto is not None:
397         bld.HEIMDAL_AUTOPROTO(heimdal_autoproto, source, options=heimdal_autoproto_options)
398     if heimdal_autoproto_private is not None:
399         bld.HEIMDAL_AUTOPROTO_PRIVATE(heimdal_autoproto_private, source)
400     if autoproto is not None:
401         bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
402     if public_headers is not None:
403         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
404     return t
405
406
407 Build.BuildContext.SAMBA_SUBSYSTEM = SAMBA_SUBSYSTEM
408
409
410 def SAMBA_GENERATOR(bld, name, rule, source, target,
411                     group='generators', enabled=True,
412                     public_headers=None,
413                     header_path=None,
414                     vars=None):
415     '''A generic source generator target'''
416
417     if not SET_TARGET_TYPE(bld, name, 'GENERATOR'):
418         return
419
420     if not enabled:
421         return
422
423     bld.SET_BUILD_GROUP(group)
424     t = bld(
425         rule=rule,
426         source=bld.EXPAND_VARIABLES(source, vars=vars),
427         target=target,
428         shell=isinstance(rule, str),
429         on_results=True,
430         before='cc',
431         ext_out='.c',
432         name=name)
433
434     if public_headers is not None:
435         bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
436     return t
437 Build.BuildContext.SAMBA_GENERATOR = SAMBA_GENERATOR
438
439
440
441 def BUILD_SUBDIR(bld, dir):
442     '''add a new set of build rules from a subdirectory'''
443     path = os.path.normpath(bld.curdir + '/' + dir)
444     cache = LOCAL_CACHE(bld, 'SUBDIR_LIST')
445     if path in cache: return
446     cache[path] = True
447     debug("build: Processing subdirectory %s" % dir)
448     bld.add_subdirs(dir)
449 Build.BuildContext.BUILD_SUBDIR = BUILD_SUBDIR
450
451
452
453 @runonce
454 def SETUP_BUILD_GROUPS(bld):
455     '''setup build groups used to ensure that the different build
456     phases happen consecutively'''
457     bld.p_ln = bld.srcnode # we do want to see all targets!
458     bld.env['USING_BUILD_GROUPS'] = True
459     bld.add_group('setup')
460     bld.add_group('build_compiler_source')
461     bld.add_group('base_libraries')
462     bld.add_group('generators')
463     bld.add_group('compiler_prototypes')
464     bld.add_group('compiler_libraries')
465     bld.add_group('build_compilers')
466     bld.add_group('build_source')
467     bld.add_group('prototypes')
468     bld.add_group('main')
469     bld.add_group('binaries')
470     bld.add_group('final')
471 Build.BuildContext.SETUP_BUILD_GROUPS = SETUP_BUILD_GROUPS
472
473
474 def SET_BUILD_GROUP(bld, group):
475     '''set the current build group'''
476     if not 'USING_BUILD_GROUPS' in bld.env:
477         return
478     bld.set_group(group)
479 Build.BuildContext.SET_BUILD_GROUP = SET_BUILD_GROUP
480
481
482
483 @conf
484 def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
485     """use timestamps instead of file contents for deps
486     this currently doesn't work"""
487     def h_file(filename):
488         import stat
489         st = os.stat(filename)
490         if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
491         m = Utils.md5()
492         m.update(str(st.st_mtime))
493         m.update(str(st.st_size))
494         m.update(filename)
495         return m.digest()
496     Utils.h_file = h_file
497
498
499
500 t = Task.simple_task_type('copy_script', 'rm -f ${LINK_TARGET} && ln -s ${SRC[0].abspath(env)} ${LINK_TARGET}',
501                           shell=True, color='PINK', ext_in='.bin')
502 t.quiet = True
503
504 @feature('copy_script')
505 @before('apply_link')
506 def copy_script(self):
507     tsk = self.create_task('copy_script', self.allnodes[0])
508     tsk.env.TARGET = self.target
509
510 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
511     '''used to copy scripts from the source tree into the build directory
512        for use by selftest'''
513
514     source = bld.path.ant_glob(pattern)
515
516     bld.SET_BUILD_GROUP('build_source')
517     for s in TO_LIST(source):
518         iname = s
519         if installname != None:
520             iname = installname
521         target = os.path.join(installdir, iname)
522         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
523         mkdir_p(tgtdir)
524         t = bld(features='copy_script',
525                 source       = s,
526                 target       = target,
527                 always       = True,
528                 install_path = None)
529         t.env.LINK_TARGET = target
530
531 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
532
533
534 def install_file(bld, destdir, file, chmod=0644, flat=False,
535                  python_fixup=False, destname=None, base_name=None):
536     '''install a file'''
537     destdir = bld.EXPAND_VARIABLES(destdir)
538     if not destname:
539         destname = file
540         if flat:
541             destname = os.path.basename(destname)
542     dest = os.path.join(destdir, destname)
543     if python_fixup:
544         # fixup the python path it will use to find Samba modules
545         inst_file = file + '.inst'
546         bld.SAMBA_GENERATOR('python_%s' % destname,
547                             rule="sed 's|\(sys.path.insert.*\)bin/python\(.*\)$|\\1${PYTHONDIR}\\2|g' < ${SRC} > ${TGT}",
548                             source=file,
549                             target=inst_file)
550         file = inst_file
551     if base_name:
552         file = os.path.join(base_name, file)
553     bld.install_as(dest, file, chmod=chmod)
554
555
556 def INSTALL_FILES(bld, destdir, files, chmod=0644, flat=False,
557                   python_fixup=False, destname=None, base_name=None):
558     '''install a set of files'''
559     for f in TO_LIST(files):
560         install_file(bld, destdir, f, chmod=chmod, flat=flat,
561                      python_fixup=python_fixup, destname=destname,
562                      base_name=base_name)
563 Build.BuildContext.INSTALL_FILES = INSTALL_FILES
564
565
566 def INSTALL_WILDCARD(bld, destdir, pattern, chmod=0644, flat=False,
567                      python_fixup=False, exclude=None, trim_path=None):
568     '''install a set of files matching a wildcard pattern'''
569     files=TO_LIST(bld.path.ant_glob(pattern))
570     if trim_path:
571         files2 = []
572         for f in files:
573             files2.append(os_path_relpath(f, trim_path))
574         files = files2
575
576     if exclude:
577         for f in files[:]:
578             if fnmatch.fnmatch(f, exclude):
579                 files.remove(f)
580     INSTALL_FILES(bld, destdir, files, chmod=chmod, flat=flat,
581                   python_fixup=python_fixup, base_name=trim_path)
582 Build.BuildContext.INSTALL_WILDCARD = INSTALL_WILDCARD
583
584
585 def PUBLIC_HEADERS(bld, public_headers, header_path=None):
586     '''install some headers
587
588     header_path may either be a string that is added to the INCLUDEDIR,
589     or it can be a dictionary of wildcard patterns which map to destination
590     directories relative to INCLUDEDIR
591     '''
592     dest = '${INCLUDEDIR}'
593     if isinstance(header_path, str):
594         dest += '/' + header_path
595     for h in TO_LIST(public_headers):
596         hdest = dest
597         if isinstance(header_path, list):
598             for (p1, dir) in header_path:
599                 found_match=False
600                 lst = TO_LIST(p1)
601                 for p2 in lst:
602                     if fnmatch.fnmatch(h, p2):
603                         if dir:
604                             hdest = os.path.join(hdest, dir)
605                         found_match=True
606                         break
607                 if found_match: break
608         if h.find(':') != -1:
609             hs=h.split(':')
610             INSTALL_FILES(bld, hdest, hs[0], flat=True, destname=hs[1])
611         else:
612             INSTALL_FILES(bld, hdest, h, flat=True)
613 Build.BuildContext.PUBLIC_HEADERS = PUBLIC_HEADERS
614
615
616 def subst_at_vars(task):
617     '''substiture @VAR@ style variables in a file'''
618     src = task.inputs[0].srcpath(task.env)
619     tgt = task.outputs[0].bldpath(task.env)
620
621     f = open(src, 'r')
622     s = f.read()
623     f.close()
624     # split on the vars
625     a = re.split('(@\w+@)', s)
626     out = []
627     for v in a:
628         if re.match('@\w+@', v):
629             vname = v[1:-1]
630             if not vname in task.env and vname.upper() in task.env:
631                 vname = vname.upper()
632             if not vname in task.env:
633                 print "Unknown substitution %s in %s" % (v, task.name)
634                 raise
635             v = task.env[vname]
636         out.append(v)
637     contents = ''.join(out)
638     f = open(tgt, 'w')
639     s = f.write(contents)
640     f.close()
641     return 0
642
643
644
645 def PKG_CONFIG_FILES(bld, pc_files, vnum=None):
646     '''install some pkg_config pc files'''
647     dest = '${PKGCONFIGDIR}'
648     dest = bld.EXPAND_VARIABLES(dest)
649     for f in TO_LIST(pc_files):
650         base=os.path.basename(f)
651         t = bld.SAMBA_GENERATOR('PKGCONFIG_%s' % base,
652                                 rule=subst_at_vars,
653                                 source=f+'.in',
654                                 target=f)
655         if vnum:
656             t.env.PACKAGE_VERSION = vnum
657         INSTALL_FILES(bld, dest, f, flat=True, destname=base)
658 Build.BuildContext.PKG_CONFIG_FILES = PKG_CONFIG_FILES
659
660
661
662 #############################################################
663 # give a nicer display when building different types of files
664 def progress_display(self, msg, fname):
665     col1 = Logs.colors(self.color)
666     col2 = Logs.colors.NORMAL
667     total = self.position[1]
668     n = len(str(total))
669     fs = '[%%%dd/%%%dd] %s %%s%%s%%s\n' % (n, n, msg)
670     return fs % (self.position[0], self.position[1], col1, fname, col2)
671
672 def link_display(self):
673     if Options.options.progress_bar != 0:
674         return Task.Task.old_display(self)
675     fname = self.outputs[0].bldpath(self.env)
676     return progress_display(self, 'Linking', fname)
677 Task.TaskBase.classes['cc_link'].display = link_display
678
679 def samba_display(self):
680     if Options.options.progress_bar != 0:
681         return Task.Task.old_display(self)
682
683     targets    = LOCAL_CACHE(self, 'TARGET_TYPE')
684     if self.name in targets:
685         target_type = targets[self.name]
686         type_map = { 'GENERATOR' : 'Generating',
687                      'PROTOTYPE' : 'Generating'
688                      }
689         if target_type in type_map:
690             return progress_display(self, type_map[target_type], self.name)
691
692     fname = self.inputs[0].bldpath(self.env)
693     if fname[0:3] == '../':
694         fname = fname[3:]
695     ext_loc = fname.rfind('.')
696     if ext_loc == -1:
697         return Task.Task.old_display(self)
698     ext = fname[ext_loc:]
699
700     ext_map = { '.idl' : 'Compiling IDL',
701                 '.et'  : 'Compiling ERRTABLE',
702                 '.asn1': 'Compiling ASN1',
703                 '.c'   : 'Compiling' }
704     if ext in ext_map:
705         return progress_display(self, ext_map[ext], fname)
706     return Task.Task.old_display(self)
707
708 Task.TaskBase.classes['Task'].old_display = Task.TaskBase.classes['Task'].display
709 Task.TaskBase.classes['Task'].display = samba_display