wafsamba: add optional keep_underscore=True to SAMBA_LIBRARY()
[obnox/samba/samba-obnox.git] / buildtools / wafsamba / wafsamba.py
index 51d85731eb84a1d94e2a064192b601c59359a1bc..bd2ca89f62ab858ed5b754cdcd0b8e3793871ba0 100644 (file)
@@ -1,41 +1,75 @@
 # a waf tool to add autoconf-like macros to the configure section
 # and for SAMBA_ macros for building libraries, binaries etc
 
-import Build, os, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs
+import Build, os, sys, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs, Constants
 from Configure import conf
 from Logs import debug
 from samba_utils import SUBST_VARS_RECURSIVE
+TaskGen.task_gen.apply_verif = Utils.nada
 
 # bring in the other samba modules
 from samba_optimisation import *
 from samba_utils import *
+from samba_version import *
 from samba_autoconf import *
 from samba_patterns import *
 from samba_pidl import *
-from samba_errtable import *
-from samba_asn1 import *
 from samba_autoproto import *
 from samba_python import *
+from samba_perl import *
 from samba_deps import *
 from samba_bundled import *
+from samba_third_party import *
+import samba_install
 import samba_conftests
+import samba_abi
+import samba_headers
+import tru64cc
+import irixcc
+import hpuxcc
+import generic_cc
+import samba_dist
+import samba_wildcard
+import stale_files
+import symbols
+import pkgconfig
+import configure_file
+
+# some systems have broken threading in python
+if os.environ.get('WAF_NOTHREADS') == '1':
+    import nothreads
 
 LIB_PATH="shared"
 
-os.putenv('PYTHONUNBUFFERED', '1')
+os.environ['PYTHONUNBUFFERED'] = '1'
+
+
+if Constants.HEXVERSION < 0x105019:
+    Logs.error('''
+Please use the version of waf that comes with Samba, not
+a system installed version. See http://wiki.samba.org/index.php/Waf
+for details.
+
+Alternatively, please run ./configure and make as usual. That will
+call the right version of waf.''')
+    sys.exit(1)
+
 
 @conf
 def SAMBA_BUILD_ENV(conf):
     '''create the samba build environment'''
-    conf.env['BUILD_DIRECTORY'] = conf.blddir
+    conf.env.BUILD_DIRECTORY = conf.blddir
     mkdir_p(os.path.join(conf.blddir, LIB_PATH))
+    mkdir_p(os.path.join(conf.blddir, LIB_PATH, "private"))
+    mkdir_p(os.path.join(conf.blddir, "modules"))
     mkdir_p(os.path.join(conf.blddir, 'python/samba/dcerpc'))
     # this allows all of the bin/shared and bin/python targets
     # to be expressed in terms of build directory paths
-    for p in ['python','shared']:
-        link_target = os.path.join(conf.blddir, 'default/' + p)
+    mkdir_p(os.path.join(conf.blddir, 'default'))
+    for (source, target) in [('shared', 'shared'), ('modules', 'modules'), ('python', 'python_modules')]:
+        link_target = os.path.join(conf.blddir, 'default/' + target)
         if not os.path.lexists(link_target):
-            os.symlink('../' + p, link_target)
+            os.symlink('../' + source, link_target)
 
     # get perl to put the blib files in the build directory
     blib_bld = os.path.join(conf.blddir, 'default/pidl/blib')
@@ -46,8 +80,6 @@ def SAMBA_BUILD_ENV(conf):
         os.unlink(blib_src)
     elif os.path.exists(blib_src):
         shutil.rmtree(blib_src)
-    os.symlink(blib_bld, blib_src)
-
 
 
 def ADD_INIT_FUNCTION(bld, subsystem, target, init_function):
@@ -69,42 +101,68 @@ def SAMBA_LIBRARY(bld, libname, source,
                   public_deps='',
                   includes='',
                   public_headers=None,
+                  public_headers_install=True,
                   header_path=None,
                   pc_files=None,
                   vnum=None,
+                  soname=None,
                   cflags='',
+                  ldflags='',
                   external_library=False,
                   realname=None,
+                  keep_underscore=False,
                   autoproto=None,
+                  autoproto_extra_source='',
                   group='main',
                   depends_on='',
                   local_include=True,
+                  global_include=True,
                   vars=None,
+                  subdir=None,
                   install_path=None,
                   install=True,
-                  needs_python=False,
+                  pyembed=False,
+                  pyext=False,
                   target_type='LIBRARY',
                   bundled_extension=True,
                   link_name=None,
+                  abi_directory=None,
+                  abi_match=None,
+                  hide_symbols=False,
+                  manpages=None,
+                  private_library=False,
+                  grouping_library=False,
+                  allow_undefined_symbols=False,
+                  allow_warnings=False,
                   enabled=True):
     '''define a Samba library'''
 
+    if LIB_MUST_BE_PRIVATE(bld, libname):
+        private_library=True
+
     if not enabled:
         SET_TARGET_TYPE(bld, libname, 'DISABLED')
         return
 
     source = bld.EXPAND_VARIABLES(source, vars=vars)
+    if subdir:
+        source = bld.SUBDIR(subdir, source)
 
     # remember empty libraries, so we can strip the dependencies
-    if (source == '') or (source == []):
+    if ((source == '') or (source == [])) and deps == '' and public_deps == '':
         SET_TARGET_TYPE(bld, libname, 'EMPTY')
         return
 
-    if target_type != 'PYTHON' and BUILTIN_LIBRARY(bld, libname):
+    if BUILTIN_LIBRARY(bld, libname):
         obj_target = libname
     else:
         obj_target = libname + '.objlist'
 
+    if group == 'libraries':
+        subsystem_group = 'main'
+    else:
+        subsystem_group = group
+
     # first create a target for building the object files for this library
     # by separating in this way, we avoid recompiling the C files
     # separately for the install library and the build library
@@ -114,15 +172,21 @@ def SAMBA_LIBRARY(bld, libname, source,
                         public_deps    = public_deps,
                         includes       = includes,
                         public_headers = public_headers,
+                        public_headers_install = public_headers_install,
                         header_path    = header_path,
                         cflags         = cflags,
-                        group          = group,
+                        group          = subsystem_group,
                         autoproto      = autoproto,
+                        autoproto_extra_source=autoproto_extra_source,
                         depends_on     = depends_on,
-                        needs_python   = needs_python,
-                        local_include  = local_include)
-
-    if libname == obj_target:
+                        hide_symbols   = hide_symbols,
+                        allow_warnings = allow_warnings,
+                        pyembed        = pyembed,
+                        pyext          = pyext,
+                        local_include  = local_include,
+                        global_include = global_include)
+
+    if BUILTIN_LIBRARY(bld, libname):
         return
 
     if not SET_TARGET_TYPE(bld, libname, target_type):
@@ -133,91 +197,106 @@ def SAMBA_LIBRARY(bld, libname, source,
     deps = TO_LIST(deps)
     deps.append(obj_target)
 
-    if needs_python:
-        bundled_name = libname
+    realname = bld.map_shlib_extension(realname, python=(target_type=='PYTHON'))
+    link_name = bld.map_shlib_extension(link_name, python=(target_type=='PYTHON'))
+
+    # we don't want any public libraries without version numbers
+    if (not private_library and target_type != 'PYTHON' and not realname):
+        if vnum is None and soname is None:
+            raise Utils.WafError("public library '%s' must have a vnum" %
+                    libname)
+        if pc_files is None:
+            raise Utils.WafError("public library '%s' must have pkg-config file" %
+                       libname)
+        if public_headers is None:
+            raise Utils.WafError("public library '%s' must have header files" %
+                       libname)
+
+    if target_type == 'PYTHON' or realname or not private_library:
+        if keep_underscore:
+            bundled_name = libname
+        else:
+            bundled_name = libname.replace('_', '-')
     else:
-        bundled_name = BUNDLED_NAME(bld, libname, bundled_extension)
+        bundled_name = PRIVATE_NAME(bld, libname, bundled_extension,
+            private_library)
+
+    ldflags = TO_LIST(ldflags)
 
-    features = 'cc cshlib'
-    if needs_python:
+    features = 'cc cshlib symlink_lib install_lib'
+    if pyext:
         features += ' pyext'
+    if pyembed:
+        features += ' pyembed'
+
+    if abi_directory:
+        features += ' abi_check'
+
+    vscript = None
+    if bld.env.HAVE_LD_VERSION_SCRIPT:
+        if private_library:
+            version = "%s_%s" % (Utils.g_module.APPNAME, Utils.g_module.VERSION)
+        elif vnum:
+            version = "%s_%s" % (libname, vnum)
+        else:
+            version = None
+        if version:
+            vscript = "%s.vscript" % libname
+            bld.ABI_VSCRIPT(libname, abi_directory, version, vscript,
+                            abi_match)
+            fullname = apply_pattern(bundled_name, bld.env.shlib_PATTERN)
+            fullpath = bld.path.find_or_declare(fullname)
+            vscriptpath = bld.path.find_or_declare(vscript)
+            if not fullpath:
+                raise Utils.WafError("unable to find fullpath for %s" % fullname)
+            if not vscriptpath:
+                raise Utils.WafError("unable to find vscript path for %s" % vscript)
+            bld.add_manual_dependency(fullpath, vscriptpath)
+            if Options.is_install:
+                # also make the .inst file depend on the vscript
+                instname = apply_pattern(bundled_name + '.inst', bld.env.shlib_PATTERN)
+                bld.add_manual_dependency(bld.path.find_or_declare(instname), bld.path.find_or_declare(vscript))
+            vscript = os.path.join(bld.path.abspath(bld.env), vscript)
 
     bld.SET_BUILD_GROUP(group)
     t = bld(
-        features        = features + ' symlink_lib',
+        features        = features,
         source          = [],
         target          = bundled_name,
-        samba_cflags    = CURRENT_CFLAGS(bld, libname, cflags),
         depends_on      = depends_on,
+        samba_ldflags   = ldflags,
         samba_deps      = deps,
         samba_includes  = includes,
+        version_script  = vscript,
         local_include   = local_include,
+        global_include  = global_include,
         vnum            = vnum,
+        soname          = soname,
         install_path    = None,
-        ldflags         = build_rpath(bld),
-        name           = libname
+        samba_inst_path = install_path,
+        name            = libname,
+        samba_realname  = realname,
+        samba_install   = install,
+        abi_directory   = "%s/%s" % (bld.path.abspath(), abi_directory),
+        abi_match       = abi_match,
+        private_library = private_library,
+        grouping_library=grouping_library,
+        allow_undefined_symbols=allow_undefined_symbols
         )
 
+    if realname and not link_name:
+        link_name = 'shared/%s' % realname
+
     if link_name:
         t.link_name = link_name
 
-    if install_path is None:
-        install_path = '${LIBDIR}'
-    install_path = SUBST_VARS_RECURSIVE(install_path, bld.env)
-
-    # we don't need the double libraries if rpath is off
-    if (bld.env.RPATH_ON_INSTALL == False and
-        bld.env.RPATH_ON_BUILD == False):
-        install_target = bundled_name
-    else:
-        install_target = bundled_name + '.inst'
-
-    if install and install_target != bundled_name:
-        # create a separate install library, which may have
-        # different rpath settings
-        SET_TARGET_TYPE(bld, install_target, target_type)
-        t = bld(
-            features        = features,
-            source          = [],
-            target          = install_target,
-            samba_cflags    = CURRENT_CFLAGS(bld, libname, cflags),
-            depends_on      = depends_on,
-            samba_deps      = deps,
-            samba_includes  = includes,
-            local_include   = local_include,
-            vnum            = vnum,
-            install_as     = bundled_name,
-            install_path    = None,
-            ldflags         = install_rpath(bld)
-            )
-
-    if install:
-        if realname:
-            install_name = realname
-            install_link = None
-            inst_name    = install_target + '.so'
-        elif vnum:
-            vnum_base = vnum.split('.')[0]
-            install_name = 'lib%s.so.%s' % (bundled_name, vnum)
-            install_link = 'lib%s.so.%s' % (bundled_name, vnum_base)
-            inst_name    = 'lib%s.so' % install_target
-        else:
-            install_name = 'lib%s.so' % bundled_name
-            install_link = None
-            inst_name    = 'lib%s.so' % install_target
-
-        bld.install_as(os.path.join(install_path, install_name), inst_name)
-        if install_link:
-            bld.symlink_as(os.path.join(install_path, install_link), install_name)
+    if pc_files is not None and not private_library:
+        bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
 
-    if autoproto is not None:
-        bld.SAMBA_AUTOPROTO(autoproto, source)
+    if (manpages is not None and 'XSLTPROC_MANPAGES' in bld.env and
+        bld.env['XSLTPROC_MANPAGES']):
+        bld.MANPAGES(manpages, install)
 
-    if public_headers is not None:
-        bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
-
-    if pc_files is not None:
-        bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
 
 Build.BuildContext.SAMBA_LIBRARY = SAMBA_LIBRARY
 
@@ -229,32 +308,56 @@ def SAMBA_BINARY(bld, binname, source,
                  public_headers=None,
                  header_path=None,
                  modules=None,
-                 installdir=None,
                  ldflags=None,
                  cflags='',
                  autoproto=None,
-                 use_hostcc=None,
+                 use_hostcc=False,
+                 use_global_deps=True,
                  compiler=None,
-                 group='binaries',
+                 group='main',
                  manpages=None,
                  local_include=True,
+                 global_include=True,
                  subsystem_name=None,
-                 needs_python=False,
+                 pyembed=False,
                  vars=None,
+                 subdir=None,
                  install=True,
-                 install_path=None):
+                 install_path=None,
+                 enabled=True):
     '''define a Samba binary'''
 
+    if not enabled:
+        SET_TARGET_TYPE(bld, binname, 'DISABLED')
+        return
+
     if not SET_TARGET_TYPE(bld, binname, 'BINARY'):
         return
 
-    features = 'cc cprogram'
-    if needs_python:
+    features = 'cc cprogram symlink_bin install_bin'
+    if pyembed:
         features += ' pyembed'
 
     obj_target = binname + '.objlist'
 
     source = bld.EXPAND_VARIABLES(source, vars=vars)
+    if subdir:
+        source = bld.SUBDIR(subdir, source)
+    source = unique_list(TO_LIST(source))
+
+    if group == 'binaries':
+        subsystem_group = 'main'
+    else:
+        subsystem_group = group
+
+    # only specify PIE flags for binaries
+    pie_cflags = cflags
+    pie_ldflags = TO_LIST(ldflags)
+    if bld.env['ENABLE_PIE'] is True:
+        pie_cflags += ' -fPIE'
+        pie_ldflags.extend(TO_LIST('-pie'))
+    if bld.env['ENABLE_RELRO'] is True:
+        pie_ldflags.extend(TO_LIST('-Wl,-z,relro,-z,now'))
 
     # first create a target for building the object files for this binary
     # by separating in this way, we avoid recompiling the C files
@@ -263,79 +366,42 @@ def SAMBA_BINARY(bld, binname, source,
                         source         = source,
                         deps           = deps,
                         includes       = includes,
-                        cflags         = cflags,
-                        group          = group,
+                        cflags         = pie_cflags,
+                        group          = subsystem_group,
                         autoproto      = autoproto,
                         subsystem_name = subsystem_name,
-                        needs_python   = needs_python,
-                        local_include  = local_include)
+                        local_include  = local_include,
+                        global_include = global_include,
+                        use_hostcc     = use_hostcc,
+                        pyext          = pyembed,
+                        use_global_deps= use_global_deps)
 
     bld.SET_BUILD_GROUP(group)
 
-    # the library itself will depend on that object target
+    # the binary itself will depend on that object target
     deps = TO_LIST(deps)
     deps.append(obj_target)
 
-    bld(
-        features       = features + ' symlink_bin',
+    t = bld(
+        features       = features,
         source         = [],
         target         = binname,
-        samba_cflags   = CURRENT_CFLAGS(bld, binname, cflags),
         samba_deps     = deps,
         samba_includes = includes,
         local_include  = local_include,
+        global_include = global_include,
         samba_modules  = modules,
         top            = True,
         samba_subsystem= subsystem_name,
         install_path   = None,
-        ldflags        = build_rpath(bld)
+        samba_inst_path= install_path,
+        samba_install  = install,
+        samba_ldflags  = pie_ldflags
         )
 
-    if install_path is None:
-        install_path = '${BINDIR}'
-    install_path = SUBST_VARS_RECURSIVE(install_path, bld.env)
-
-    # we don't need the double binaries if rpath is off
-    if (bld.env.RPATH_ON_INSTALL == False and
-        bld.env.RPATH_ON_BUILD == False):
-        install_target = binname
-    else:
-        install_target = binname + '.inst'
-
-    if install and install_target != binname:
-        # we create a separate 'install' binary, which
-        # will have different rpath settings
-        SET_TARGET_TYPE(bld, install_target, 'BINARY')
-        t = bld(
-            features       = features,
-            source         = [],
-            target         = install_target,
-            samba_cflags   = CURRENT_CFLAGS(bld, binname, cflags),
-            samba_deps     = deps,
-            samba_includes = includes,
-            local_include  = local_include,
-            samba_modules  = modules,
-            top            = True,
-            samba_subsystem= subsystem_name,
-            install_path   = None,
-            ldflags        = install_rpath(bld)
-            )
-
-    if install:
-        bld.install_as(os.path.join(install_path, binname),
-                       install_target,
-                       chmod=0755)
-
-    # setup the subsystem_name as an alias for the real
-    # binary name, so it can be found when expanding
-    # subsystem dependencies
-    if subsystem_name is not None:
-        bld.TARGET_ALIAS(subsystem_name, binname)
+    if manpages is not None and 'XSLTPROC_MANPAGES' in bld.env and bld.env['XSLTPROC_MANPAGES']:
+        bld.MANPAGES(manpages, install)
 
-    if autoproto is not None:
-        bld.SAMBA_AUTOPROTO(autoproto, source)
-    if public_headers is not None:
-        bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
 Build.BuildContext.SAMBA_BINARY = SAMBA_BINARY
 
 
@@ -345,63 +411,90 @@ def SAMBA_MODULE(bld, modname, source,
                  includes='',
                  subsystem=None,
                  init_function=None,
+                 module_init_name='samba_init_module',
                  autoproto=None,
                  autoproto_extra_source='',
-                 aliases=None,
                  cflags='',
                  internal_module=True,
                  local_include=True,
+                 global_include=True,
                  vars=None,
-                 enabled=True):
+                 subdir=None,
+                 enabled=True,
+                 pyembed=False,
+                 manpages=None,
+                 allow_undefined_symbols=False,
+                 allow_warnings=False
+                 ):
     '''define a Samba module.'''
 
-    # we add the init function regardless of whether the module
-    # is enabled or not, as we need to generate a null list if
-    # all disabled
-    bld.ADD_INIT_FUNCTION(subsystem, modname, init_function)
+    source = bld.EXPAND_VARIABLES(source, vars=vars)
+    if subdir:
+        source = bld.SUBDIR(subdir, source)
 
     if internal_module or BUILTIN_LIBRARY(bld, modname):
-        # treat internal modules as subsystems for now
-        SAMBA_SUBSYSTEM(bld, modname, source,
-                        deps=deps,
-                        includes=includes,
-                        autoproto=autoproto,
-                        autoproto_extra_source=autoproto_extra_source,
-                        cflags=cflags,
-                        local_include=local_include,
-                        enabled=enabled)
+        # Do not create modules for disabled subsystems
+        if subsystem and GET_TARGET_TYPE(bld, subsystem) == 'DISABLED':
+            return
+        bld.SAMBA_SUBSYSTEM(modname, source,
+                    deps=deps,
+                    includes=includes,
+                    autoproto=autoproto,
+                    autoproto_extra_source=autoproto_extra_source,
+                    cflags=cflags,
+                    local_include=local_include,
+                    global_include=global_include,
+                    allow_warnings=allow_warnings,
+                    enabled=enabled)
+
+        bld.ADD_INIT_FUNCTION(subsystem, modname, init_function)
         return
 
     if not enabled:
         SET_TARGET_TYPE(bld, modname, 'DISABLED')
         return
 
-    source = bld.EXPAND_VARIABLES(source, vars=vars)
-
-    # remember empty modules, so we can strip the dependencies
-    if (source == '') or (source == []):
-        SET_TARGET_TYPE(bld, modname, 'EMPTY')
+    # Do not create modules for disabled subsystems
+    if subsystem and GET_TARGET_TYPE(bld, subsystem) == 'DISABLED':
         return
 
-    if not SET_TARGET_TYPE(bld, modname, 'MODULE'):
-        return
+    obj_target = modname + '.objlist'
 
+    realname = modname
     if subsystem is not None:
         deps += ' ' + subsystem
+        while realname.startswith("lib"+subsystem+"_"):
+            realname = realname[len("lib"+subsystem+"_"):]
+        while realname.startswith(subsystem+"_"):
+            realname = realname[len(subsystem+"_"):]
+
+    realname = bld.make_libname(realname)
+    while realname.startswith("lib"):
+        realname = realname[len("lib"):]
+
+    build_link_name = "modules/%s/%s" % (subsystem, realname)
+
+    if init_function:
+        cflags += " -D%s=%s" % (init_function, module_init_name)
+
+    bld.SAMBA_LIBRARY(modname,
+                      source,
+                      deps=deps,
+                      includes=includes,
+                      cflags=cflags,
+                      realname = realname,
+                      autoproto = autoproto,
+                      local_include=local_include,
+                      global_include=global_include,
+                      vars=vars,
+                      link_name=build_link_name,
+                      install_path="${MODULESDIR}/%s" % subsystem,
+                      pyembed=pyembed,
+                      manpages=manpages,
+                      allow_undefined_symbols=allow_undefined_symbols,
+                      allow_warnings=allow_warnings
+                      )
 
-    bld.SET_BUILD_GROUP('main')
-    bld(
-        features       = 'cc',
-        source         = source,
-        target         = modname,
-        samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
-        samba_includes = includes,
-        local_include  = local_include,
-        samba_deps     = TO_LIST(deps)
-        )
-
-    if autoproto is not None:
-        bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
 
 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
 
@@ -412,23 +505,28 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                     public_deps='',
                     includes='',
                     public_headers=None,
+                    public_headers_install=True,
                     header_path=None,
                     cflags='',
                     cflags_end=None,
                     group='main',
-                    init_function_sentinal=None,
-                    heimdal_autoproto=None,
-                    heimdal_autoproto_options=None,
-                    heimdal_autoproto_private=None,
+                    init_function_sentinel=None,
                     autoproto=None,
                     autoproto_extra_source='',
                     depends_on='',
                     local_include=True,
                     local_include_first=True,
+                    global_include=True,
                     subsystem_name=None,
                     enabled=True,
+                    use_hostcc=False,
+                    use_global_deps=True,
                     vars=None,
-                    needs_python=False):
+                    subdir=None,
+                    hide_symbols=False,
+                    allow_warnings=False,
+                    pyext=False,
+                    pyembed=False):
     '''define a Samba subsystem'''
 
     if not enabled:
@@ -436,7 +534,7 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
         return
 
     # remember empty subsystems, so we can strip the dependencies
-    if (source == '') or (source == []):
+    if ((source == '') or (source == [])) and deps == '' and public_deps == '':
         SET_TARGET_TYPE(bld, modname, 'EMPTY')
         return
 
@@ -444,50 +542,60 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
         return
 
     source = bld.EXPAND_VARIABLES(source, vars=vars)
+    if subdir:
+        source = bld.SUBDIR(subdir, source)
+    source = unique_list(TO_LIST(source))
 
     deps += ' ' + public_deps
 
     bld.SET_BUILD_GROUP(group)
 
     features = 'cc'
-    if needs_python:
+    if pyext:
         features += ' pyext'
+    if pyembed:
+        features += ' pyembed'
 
     t = bld(
         features       = features,
         source         = source,
         target         = modname,
-        samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags),
+        samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags,
+                                        allow_warnings=allow_warnings,
+                                        hide_symbols=hide_symbols),
         depends_on     = depends_on,
         samba_deps     = TO_LIST(deps),
         samba_includes = includes,
         local_include  = local_include,
         local_include_first  = local_include_first,
-        samba_subsystem= subsystem_name
+        global_include = global_include,
+        samba_subsystem= subsystem_name,
+        samba_use_hostcc = use_hostcc,
+        samba_use_global_deps = use_global_deps,
         )
 
     if cflags_end is not None:
         t.samba_cflags.extend(TO_LIST(cflags_end))
 
-    if heimdal_autoproto is not None:
-        bld.HEIMDAL_AUTOPROTO(heimdal_autoproto, source, options=heimdal_autoproto_options)
-    if heimdal_autoproto_private is not None:
-        bld.HEIMDAL_AUTOPROTO_PRIVATE(heimdal_autoproto_private, source)
     if autoproto is not None:
-        bld.SAMBA_AUTOPROTO(autoproto, source + ' ' + autoproto_extra_source)
+        bld.SAMBA_AUTOPROTO(autoproto, source + TO_LIST(autoproto_extra_source))
     if public_headers is not None:
-        bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
+        bld.PUBLIC_HEADERS(public_headers, header_path=header_path,
+                           public_headers_install=public_headers_install)
     return t
 
 
 Build.BuildContext.SAMBA_SUBSYSTEM = SAMBA_SUBSYSTEM
 
 
-def SAMBA_GENERATOR(bld, name, rule, source, target,
+def SAMBA_GENERATOR(bld, name, rule, source='', target='',
                     group='generators', enabled=True,
                     public_headers=None,
+                    public_headers_install=True,
                     header_path=None,
-                    vars=None):
+                    vars=None,
+                    dep_vars=[],
+                    always=False):
     '''A generic source generator target'''
 
     if not SET_TARGET_TYPE(bld, name, 'GENERATOR'):
@@ -496,6 +604,9 @@ def SAMBA_GENERATOR(bld, name, rule, source, target,
     if not enabled:
         return
 
+    dep_vars.append('ruledeps')
+    dep_vars.append('SAMBA_GENERATOR_VARS')
+
     bld.SET_BUILD_GROUP(group)
     t = bld(
         rule=rule,
@@ -505,27 +616,25 @@ def SAMBA_GENERATOR(bld, name, rule, source, target,
         on_results=True,
         before='cc',
         ext_out='.c',
+        samba_type='GENERATOR',
+        dep_vars = dep_vars,
         name=name)
 
+    if vars is None:
+        vars = {}
+    t.env.SAMBA_GENERATOR_VARS = vars
+
+    if always:
+        t.always = True
+
     if public_headers is not None:
-        bld.PUBLIC_HEADERS(public_headers, header_path=header_path)
+        bld.PUBLIC_HEADERS(public_headers, header_path=header_path,
+                           public_headers_install=public_headers_install)
     return t
 Build.BuildContext.SAMBA_GENERATOR = SAMBA_GENERATOR
 
 
 
-def BUILD_SUBDIR(bld, dir):
-    '''add a new set of build rules from a subdirectory'''
-    path = os.path.normpath(bld.curdir + '/' + dir)
-    cache = LOCAL_CACHE(bld, 'SUBDIR_LIST')
-    if path in cache: return
-    cache[path] = True
-    debug("build: Processing subdirectory %s" % dir)
-    bld.add_subdirs(dir)
-Build.BuildContext.BUILD_SUBDIR = BUILD_SUBDIR
-
-
-
 @runonce
 def SETUP_BUILD_GROUPS(bld):
     '''setup build groups used to ensure that the different build
@@ -534,6 +643,7 @@ def SETUP_BUILD_GROUPS(bld):
     bld.env['USING_BUILD_GROUPS'] = True
     bld.add_group('setup')
     bld.add_group('build_compiler_source')
+    bld.add_group('vscripts')
     bld.add_group('base_libraries')
     bld.add_group('generators')
     bld.add_group('compiler_prototypes')
@@ -541,8 +651,10 @@ def SETUP_BUILD_GROUPS(bld):
     bld.add_group('build_compilers')
     bld.add_group('build_source')
     bld.add_group('prototypes')
+    bld.add_group('headers')
     bld.add_group('main')
-    bld.add_group('binaries')
+    bld.add_group('symbolcheck')
+    bld.add_group('syslibcheck')
     bld.add_group('final')
 Build.BuildContext.SETUP_BUILD_GROUPS = SETUP_BUILD_GROUPS
 
@@ -572,73 +684,6 @@ def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
     Utils.h_file = h_file
 
 
-
-##############################
-# handle the creation of links for libraries and binaries
-# note that we use a relative symlink path to allow the whole tree
-# to me moved/copied elsewhere without breaking the links
-t = Task.simple_task_type('symlink_lib', 'rm -f ${LINK_TARGET} && ln -s ${LINK_SOURCE} ${LINK_TARGET}',
-                          shell=True, color='PINK', ext_in='.bin')
-t.quiet = True
-
-@feature('symlink_lib')
-@after('apply_link')
-def symlink_lib(self):
-    '''symlink a shared lib'''
-    tsk = self.create_task('symlink_lib', self.link_task.outputs[0])
-
-    # calculat the link target and put it in the environment
-    soext=""
-    vnum = getattr(self, 'vnum', None)
-    if vnum is not None:
-        soext = '.' + vnum.split('.')[0]
-
-    link_target = getattr(self, 'link_name', '')
-    if link_target == '':
-        link_target = '%s/lib%s.so%s' % (LIB_PATH, self.target, soext)
-
-
-    link_source = os_path_relpath(self.link_task.outputs[0].abspath(self.env),
-                                  os.path.join(self.env.BUILD_DIRECTORY, link_target))
-
-    tsk.env.LINK_TARGET = link_target
-    tsk.env.LINK_SOURCE = link_source[3:]
-    debug('task_gen: LINK for %s is %s -> %s',
-          self.name, tsk.env.LINK_SOURCE, tsk.env.LINK_TARGET)
-
-
-t = Task.simple_task_type('symlink_bin', 'rm -f ${BIN_TARGET} && ln -s ${SRC} ${BIN_TARGET}',
-                          shell=True, color='PINK', ext_in='.bin')
-t.quiet = True
-
-@feature('symlink_bin')
-@after('apply_link')
-def symlink_bin(self):
-    '''symlink a binary'''
-    if Options.is_install:
-        # we don't want to copy the install binary, as
-        # that has the install rpath, not the build rpath
-        # The rpath of the binaries in bin/default/foo/blah is different
-        # during the install phase, as distros insist on not using rpath in installed binaries
-        return
-    tsk = self.create_task('symlink_bin', self.link_task.outputs[0])
-
-    tsk.env.BIN_TARGET = self.target
-    debug('task_gen: BIN_TARGET for %s is %s', self.name, tsk.env.BIN_TARGET)
-
-
-
-
-t = Task.simple_task_type('copy_script', 'rm -f ${LINK_TARGET} && ln -s ${SRC[0].abspath(env)} ${LINK_TARGET}',
-                          shell=True, color='PINK', ext_in='.bin')
-t.quiet = True
-
-@feature('copy_script')
-@before('apply_link')
-def copy_script(self):
-    tsk = self.create_task('copy_script', self.allnodes[0])
-    tsk.env.TARGET = self.target
-
 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
     '''used to copy scripts from the source tree into the build directory
        for use by selftest'''
@@ -648,23 +693,86 @@ def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
     bld.SET_BUILD_GROUP('build_source')
     for s in TO_LIST(source):
         iname = s
-        if installname != None:
+        if installname is not None:
             iname = installname
         target = os.path.join(installdir, iname)
         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
         mkdir_p(tgtdir)
-        t = bld(features='copy_script',
-                source       = s,
-                target       = target,
-                always       = True,
-                install_path = None)
-        t.env.LINK_TARGET = target
-
+        link_src = os.path.normpath(os.path.join(bld.curdir, s))
+        link_dst = os.path.join(tgtdir, os.path.basename(iname))
+        if os.path.islink(link_dst) and os.readlink(link_dst) == link_src:
+            continue
+        if os.path.exists(link_dst):
+            os.unlink(link_dst)
+        Logs.info("symlink: %s -> %s/%s" % (s, installdir, iname))
+        os.symlink(link_src, link_dst)
 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
 
 
-def install_file(bld, destdir, file, chmod=0644, flat=False,
-                 python_fixup=False, destname=None):
+def copy_and_fix_python_path(task):
+    pattern='sys.path.insert(0, "bin/python")'
+    if task.env["PYTHONARCHDIR"] in sys.path and task.env["PYTHONDIR"] in sys.path:
+        replacement = ""
+    elif task.env["PYTHONARCHDIR"] == task.env["PYTHONDIR"]:
+        replacement="""sys.path.insert(0, "%s")""" % task.env["PYTHONDIR"]
+    else:
+        replacement="""sys.path.insert(0, "%s")
+sys.path.insert(1, "%s")""" % (task.env["PYTHONARCHDIR"], task.env["PYTHONDIR"])
+
+    if task.env["PYTHON"][0] == "/":
+        replacement_shebang = "#!%s\n" % task.env["PYTHON"]
+    else:
+        replacement_shebang = "#!/usr/bin/env %s\n" % task.env["PYTHON"]
+
+    installed_location=task.outputs[0].bldpath(task.env)
+    source_file = open(task.inputs[0].srcpath(task.env))
+    installed_file = open(installed_location, 'w')
+    lineno = 0
+    for line in source_file:
+        newline = line
+        if (lineno == 0 and task.env["PYTHON_SPECIFIED"] is True and
+                line[:2] == "#!"):
+            newline = replacement_shebang
+        elif pattern in line:
+            newline = line.replace(pattern, replacement)
+        installed_file.write(newline)
+        lineno = lineno + 1
+    installed_file.close()
+    os.chmod(installed_location, 0755)
+    return 0
+
+def copy_and_fix_perl_path(task):
+    pattern='use lib "$RealBin/lib";'
+
+    replacement = ""
+    if not task.env["PERL_LIB_INSTALL_DIR"] in task.env["PERL_INC"]:
+         replacement = 'use lib "%s";' % task.env["PERL_LIB_INSTALL_DIR"]
+
+    if task.env["PERL"][0] == "/":
+        replacement_shebang = "#!%s\n" % task.env["PERL"]
+    else:
+        replacement_shebang = "#!/usr/bin/env %s\n" % task.env["PERL"]
+
+    installed_location=task.outputs[0].bldpath(task.env)
+    source_file = open(task.inputs[0].srcpath(task.env))
+    installed_file = open(installed_location, 'w')
+    lineno = 0
+    for line in source_file:
+        newline = line
+        if lineno == 0 and task.env["PERL_SPECIFIED"] == True and line[:2] == "#!":
+            newline = replacement_shebang
+        elif pattern in line:
+            newline = line.replace(pattern, replacement)
+        installed_file.write(newline)
+        lineno = lineno + 1
+    installed_file.close()
+    os.chmod(installed_location, 0755)
+    return 0
+
+
+def install_file(bld, destdir, file, chmod=MODE_644, flat=False,
+                 python_fixup=False, perl_fixup=False,
+                 destname=None, base_name=None):
     '''install a file'''
     destdir = bld.EXPAND_VARIABLES(destdir)
     if not destname:
@@ -673,113 +781,103 @@ def install_file(bld, destdir, file, chmod=0644, flat=False,
             destname = os.path.basename(destname)
     dest = os.path.join(destdir, destname)
     if python_fixup:
-        # fixup the python path it will use to find Samba modules
+        # fix the path python will use to find Samba modules
         inst_file = file + '.inst'
         bld.SAMBA_GENERATOR('python_%s' % destname,
-                            rule="sed 's|\(sys.path.insert.*\)bin/python\(.*\)$|\\1${PYTHONDIR}\\2|g' < ${SRC} > ${TGT}",
+                            rule=copy_and_fix_python_path,
+                            dep_vars=["PYTHON","PYTHON_SPECIFIED","PYTHONDIR","PYTHONARCHDIR"],
                             source=file,
                             target=inst_file)
         file = inst_file
+    if perl_fixup:
+        # fix the path perl will use to find Samba modules
+        inst_file = file + '.inst'
+        bld.SAMBA_GENERATOR('perl_%s' % destname,
+                            rule=copy_and_fix_perl_path,
+                            dep_vars=["PERL","PERL_SPECIFIED","PERL_LIB_INSTALL_DIR"],
+                            source=file,
+                            target=inst_file)
+        file = inst_file
+    if base_name:
+        file = os.path.join(base_name, file)
     bld.install_as(dest, file, chmod=chmod)
 
 
-def INSTALL_FILES(bld, destdir, files, chmod=0644, flat=False,
-                  python_fixup=False, destname=None):
+def INSTALL_FILES(bld, destdir, files, chmod=MODE_644, flat=False,
+                  python_fixup=False, perl_fixup=False,
+                  destname=None, base_name=None):
     '''install a set of files'''
     for f in TO_LIST(files):
         install_file(bld, destdir, f, chmod=chmod, flat=flat,
-                     python_fixup=python_fixup, destname=destname)
+                     python_fixup=python_fixup, perl_fixup=perl_fixup,
+                     destname=destname, base_name=base_name)
 Build.BuildContext.INSTALL_FILES = INSTALL_FILES
 
 
-def INSTALL_WILDCARD(bld, destdir, pattern, chmod=0644, flat=False,
-                     python_fixup=False, exclude=None):
+def INSTALL_WILDCARD(bld, destdir, pattern, chmod=MODE_644, flat=False,
+                     python_fixup=False, exclude=None, trim_path=None):
     '''install a set of files matching a wildcard pattern'''
     files=TO_LIST(bld.path.ant_glob(pattern))
+    if trim_path:
+        files2 = []
+        for f in files:
+            files2.append(os_path_relpath(f, trim_path))
+        files = files2
+
     if exclude:
         for f in files[:]:
             if fnmatch.fnmatch(f, exclude):
                 files.remove(f)
-    INSTALL_FILES(bld, destdir, files, chmod=chmod, flat=flat, python_fixup=python_fixup)
+    INSTALL_FILES(bld, destdir, files, chmod=chmod, flat=flat,
+                  python_fixup=python_fixup, base_name=trim_path)
 Build.BuildContext.INSTALL_WILDCARD = INSTALL_WILDCARD
 
 
-def PUBLIC_HEADERS(bld, public_headers, header_path=None):
-    '''install some headers
-
-    header_path may either be a string that is added to the INCLUDEDIR,
-    or it can be a dictionary of wildcard patterns which map to destination
-    directories relative to INCLUDEDIR
-    '''
-    dest = '${INCLUDEDIR}'
-    if isinstance(header_path, str):
-        dest += '/' + header_path
-    for h in TO_LIST(public_headers):
-        hdest = dest
-        if isinstance(header_path, list):
-            for (p1, dir) in header_path:
-                found_match=False
-                lst = TO_LIST(p1)
-                for p2 in lst:
-                    if fnmatch.fnmatch(h, p2):
-                        if dir:
-                            hdest = os.path.join(hdest, dir)
-                        found_match=True
-                        break
-                if found_match: break
-        if h.find(':') != -1:
-            hs=h.split(':')
-            INSTALL_FILES(bld, hdest, hs[0], flat=True, destname=hs[1])
-        else:
-            INSTALL_FILES(bld, hdest, h, flat=True)
-Build.BuildContext.PUBLIC_HEADERS = PUBLIC_HEADERS
-
-
-def subst_at_vars(task):
-    '''substiture @VAR@ style variables in a file'''
-    src = task.inputs[0].srcpath(task.env)
-    tgt = task.outputs[0].bldpath(task.env)
-
-    f = open(src, 'r')
-    s = f.read()
-    f.close()
-    # split on the vars
-    a = re.split('(@\w+@)', s)
-    out = []
-    for v in a:
-        if re.match('@\w+@', v):
-            vname = v[1:-1]
-            if not vname in task.env and vname.upper() in task.env:
-                vname = vname.upper()
-            if not vname in task.env:
-                print "Unknown substitution %s in %s" % (v, task.name)
-                raise
-            v = task.env[vname]
-        out.append(v)
-    contents = ''.join(out)
-    f = open(tgt, 'w')
-    s = f.write(contents)
-    f.close()
-    return 0
-
-
-
-def PKG_CONFIG_FILES(bld, pc_files, vnum=None):
-    '''install some pkg_config pc files'''
-    dest = '${PKGCONFIGDIR}'
-    dest = bld.EXPAND_VARIABLES(dest)
-    for f in TO_LIST(pc_files):
-        base=os.path.basename(f)
-        t = bld.SAMBA_GENERATOR('PKGCONFIG_%s' % base,
-                                rule=subst_at_vars,
-                                source=f+'.in',
-                                target=f)
-        if vnum:
-            t.env.PACKAGE_VERSION = vnum
-        INSTALL_FILES(bld, dest, f, flat=True, destname=base)
-Build.BuildContext.PKG_CONFIG_FILES = PKG_CONFIG_FILES
-
-
+def INSTALL_DIRS(bld, destdir, dirs):
+    '''install a set of directories'''
+    destdir = bld.EXPAND_VARIABLES(destdir)
+    dirs = bld.EXPAND_VARIABLES(dirs)
+    for d in TO_LIST(dirs):
+        bld.install_dir(os.path.join(destdir, d))
+Build.BuildContext.INSTALL_DIRS = INSTALL_DIRS
+
+
+def MANPAGES(bld, manpages, install):
+    '''build and install manual pages'''
+    bld.env.MAN_XSL = 'http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl'
+    for m in manpages.split():
+        source = m + '.xml'
+        bld.SAMBA_GENERATOR(m,
+                            source=source,
+                            target=m,
+                            group='final',
+                            rule='${XSLTPROC} --xinclude -o ${TGT} --nonet ${MAN_XSL} ${SRC}'
+                            )
+        if install:
+            bld.INSTALL_FILES('${MANDIR}/man%s' % m[-1], m, flat=True)
+Build.BuildContext.MANPAGES = MANPAGES
+
+def SAMBAMANPAGES(bld, manpages, extra_source=None):
+    '''build and install manual pages'''
+    bld.env.SAMBA_EXPAND_XSL = bld.srcnode.abspath() + '/docs-xml/xslt/expand-sambadoc.xsl'
+    bld.env.SAMBA_MAN_XSL = bld.srcnode.abspath() + '/docs-xml/xslt/man.xsl'
+    bld.env.SAMBA_CATALOGS = 'file:///etc/xml/catalog file:///usr/local/share/xml/catalog file://' + bld.srcnode.abspath() + '/bin/default/docs-xml/build/catalog.xml'
+
+    for m in manpages.split():
+        source = m + '.xml'
+        if extra_source is not None:
+            source = [source, extra_source]
+        bld.SAMBA_GENERATOR(m,
+                            source=source,
+                            target=m,
+                            group='final',
+                            rule='''XML_CATALOG_FILES="${SAMBA_CATALOGS}"
+                                    export XML_CATALOG_FILES
+                                    ${XSLTPROC} --xinclude --stringparam noreference 0 -o ${TGT}.xml --nonet ${SAMBA_EXPAND_XSL} ${SRC[0].abspath(env)}
+                                    ${XSLTPROC} --nonet -o ${TGT} ${SAMBA_MAN_XSL} ${TGT}.xml'''
+                            )
+        bld.INSTALL_FILES('${MANDIR}/man%s' % m[-1], m, flat=True)
+Build.BuildContext.SAMBAMANPAGES = SAMBAMANPAGES
 
 #############################################################
 # give a nicer display when building different types of files
@@ -801,6 +899,19 @@ Task.TaskBase.classes['cc_link'].display = link_display
 def samba_display(self):
     if Options.options.progress_bar != 0:
         return Task.Task.old_display(self)
+
+    targets    = LOCAL_CACHE(self, 'TARGET_TYPE')
+    if self.name in targets:
+        target_type = targets[self.name]
+        type_map = { 'GENERATOR' : 'Generating',
+                     'PROTOTYPE' : 'Generating'
+                     }
+        if target_type in type_map:
+            return progress_display(self, type_map[target_type], self.name)
+
+    if len(self.inputs) == 0:
+        return Task.Task.old_display(self)
+
     fname = self.inputs[0].bldpath(self.env)
     if fname[0:3] == '../':
         fname = fname[3:]
@@ -819,3 +930,15 @@ def samba_display(self):
 
 Task.TaskBase.classes['Task'].old_display = Task.TaskBase.classes['Task'].display
 Task.TaskBase.classes['Task'].display = samba_display
+
+
+@after('apply_link')
+@feature('cshlib')
+def apply_bundle_remove_dynamiclib_patch(self):
+    if self.env['MACBUNDLE'] or getattr(self,'mac_bundle',False):
+        if not getattr(self,'vnum',None):
+            try:
+                self.env['LINKFLAGS'].remove('-dynamiclib')
+                self.env['LINKFLAGS'].remove('-single_module')
+            except ValueError:
+                pass