third_party: Update waf to version 2.0.24
[bbaumbach/samba-autobuild/.git] / buildtools / wafsamba / wafsamba.py
index a8e122c95752ee581faf058ffb5d8ee61ba18e68..79fe8b5e5758aab34a07ed215a57c2ce76b10e2c 100644 (file)
@@ -1,15 +1,16 @@
 # a waf tool to add autoconf-like macros to the configure section
 # and for SAMBA_ macros for building libraries, binaries etc
 
-import Build, os, sys, Options, Task, Utils, cc, TaskGen, fnmatch, re, shutil, Logs, Constants
-from Configure import conf
-from Logs import debug
+import os, sys, re, shutil, fnmatch
+from waflib import Build, Options, Task, Utils, TaskGen, Logs, Context, Errors
+from waflib.Configure import conf
+from waflib.Logs import debug
 from samba_utils import SUBST_VARS_RECURSIVE
 TaskGen.task_gen.apply_verif = Utils.nada
 
 # bring in the other samba modules
-from samba_optimisation import *
 from samba_utils import *
+from samba_utils import symlink
 from samba_version import *
 from samba_autoconf import *
 from samba_patterns import *
@@ -25,27 +26,19 @@ import samba_install
 import samba_conftests
 import samba_abi
 import samba_headers
-import tru64cc
-import irixcc
-import hpuxcc
 import generic_cc
 import samba_dist
 import samba_wildcard
-import stale_files
 import symbols
 import pkgconfig
 import configure_file
-
-# some systems have broken threading in python
-if os.environ.get('WAF_NOTHREADS') == '1':
-    import nothreads
+import samba_waf18
 
 LIB_PATH="shared"
 
 os.environ['PYTHONUNBUFFERED'] = '1'
 
-
-if Constants.HEXVERSION < 0x105019:
+if Context.HEXVERSION not in (0x2001800,):
     Logs.error('''
 Please use the version of waf that comes with Samba, not
 a system installed version. See http://wiki.samba.org/index.php/Waf
@@ -55,26 +48,26 @@ Alternatively, please run ./configure and make as usual. That will
 call the right version of waf.''')
     sys.exit(1)
 
-
 @conf
 def SAMBA_BUILD_ENV(conf):
     '''create the samba build environment'''
-    conf.env.BUILD_DIRECTORY = conf.blddir
-    mkdir_p(os.path.join(conf.blddir, LIB_PATH))
-    mkdir_p(os.path.join(conf.blddir, LIB_PATH, "private"))
-    mkdir_p(os.path.join(conf.blddir, "modules"))
-    mkdir_p(os.path.join(conf.blddir, 'python/samba/dcerpc'))
+    conf.env.BUILD_DIRECTORY = conf.bldnode.abspath()
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, LIB_PATH))
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, LIB_PATH, "private"))
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, "modules"))
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, "plugins"))
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, 'python/samba/dcerpc'))
     # this allows all of the bin/shared and bin/python targets
     # to be expressed in terms of build directory paths
-    mkdir_p(os.path.join(conf.blddir, 'default'))
-    for (source, target) in [('shared', 'shared'), ('modules', 'modules'), ('python', 'python_modules')]:
-        link_target = os.path.join(conf.blddir, 'default/' + target)
+    mkdir_p(os.path.join(conf.env.BUILD_DIRECTORY, 'default'))
+    for (source, target) in [('shared', 'shared'), ('modules', 'modules'), ('plugins', 'plugins'), ('python', 'python')]:
+        link_target = os.path.join(conf.env.BUILD_DIRECTORY, 'default/' + target)
         if not os.path.lexists(link_target):
-            os.symlink('../' + source, link_target)
+            symlink('../' + source, link_target)
 
     # get perl to put the blib files in the build directory
-    blib_bld = os.path.join(conf.blddir, 'default/pidl/blib')
-    blib_src = os.path.join(conf.srcdir, 'pidl/blib')
+    blib_bld = os.path.join(conf.env.BUILD_DIRECTORY, 'default/pidl/blib')
+    blib_src = os.path.join(conf.srcnode.abspath(), 'pidl/blib')
     mkdir_p(blib_bld + '/man1')
     mkdir_p(blib_bld + '/man3')
     if os.path.islink(blib_src):
@@ -106,11 +99,13 @@ def SAMBA_LIBRARY(bld, libname, source,
                   includes='',
                   public_headers=None,
                   public_headers_install=True,
+                  private_headers=None,
                   header_path=None,
                   pc_files=None,
                   vnum=None,
                   soname=None,
                   cflags='',
+                  cflags_end=None,
                   ldflags='',
                   external_library=False,
                   realname=None,
@@ -128,25 +123,57 @@ def SAMBA_LIBRARY(bld, libname, source,
                   pyembed=False,
                   pyext=False,
                   target_type='LIBRARY',
-                  bundled_extension=False,
                   bundled_name=None,
                   link_name=None,
                   abi_directory=None,
                   abi_match=None,
+                  orig_vscript_map=None,
                   hide_symbols=False,
                   manpages=None,
                   private_library=False,
                   grouping_library=False,
+                  require_builtin_deps=False,
+                  provide_builtin_linking=False,
+                  builtin_cflags='',
                   allow_undefined_symbols=False,
                   allow_warnings=False,
                   enabled=True):
     '''define a Samba library'''
 
-    if pyembed and bld.env['IS_EXTRA_PYTHON']:
-        public_headers = pc_files = None
-
-    if LIB_MUST_BE_PRIVATE(bld, libname):
-        private_library=True
+    # We support:
+    # - LIBRARY: this can be use to link via -llibname
+    # - MODULE:  this is module from SAMBA_MODULE()
+    # - PLUGIN:  this is plugin for external consumers to be
+    #            loaded via dlopen()
+    # - PYTHON:  a python C binding library
+    #
+    if target_type not in ['LIBRARY', 'MODULE', 'PLUGIN', 'PYTHON']:
+        raise Errors.WafError("target_type[%s] not supported in SAMBA_LIBRARY('%s')" %
+                              (target_type, libname))
+
+    if require_builtin_deps:
+        # For now we only support require_builtin_deps only for libraries, plugins
+        if target_type not in ['LIBRARY', 'PLUGIN']:
+            raise Errors.WafError("target_type[%s] not supported SAMBA_LIBRARY('%s', require_builtin_deps=True)" %
+                                  (target_type, libname))
+
+    if private_library and public_headers:
+        raise Errors.WafError("private library '%s' must not have public header files" %
+                             libname)
+
+    if orig_vscript_map and not private_library:
+        raise Errors.WafError("public library '%s' must not have orig_vscript_map" %
+                             libname)
+
+    if orig_vscript_map and abi_directory:
+        raise Errors.WafError("private library '%s' with orig_vscript_map must not have abi_directory" %
+                             libname)
+    if orig_vscript_map and abi_match:
+        raise Errors.WafError("private library '%s' with orig_vscript_map must not have abi_match" %
+                             libname)
+
+    if LIB_MUST_BE_PRIVATE(bld, libname) and target_type not in ['PLUGIN']:
+        private_library = True
 
     if not enabled:
         SET_TARGET_TYPE(bld, libname, 'DISABLED')
@@ -167,10 +194,26 @@ def SAMBA_LIBRARY(bld, libname, source,
                             target=empty_c)
         source=empty_c
 
+    samba_deps = deps + ' ' + public_deps
+    samba_deps = TO_LIST(samba_deps)
+
     if BUILTIN_LIBRARY(bld, libname):
-        obj_target = libname
+        builtin_target = libname + '.builtin.objlist'
+        builtin_cflags_end = '-D_PUBLIC_=_PRIVATE_'
+        empty_target = libname
+        obj_target = None
     else:
+        if provide_builtin_linking:
+            builtin_target = libname + '.builtin.objlist'
+            builtin_cflags_end = '-D_PUBLIC_=_PRIVATE_'
+        else:
+            builtin_target = None
+        empty_target = None
         obj_target = libname + '.objlist'
+        if require_builtin_deps:
+            # hide the builtin deps from the callers
+            samba_deps = TO_LIST('')
+        dep_target = obj_target
 
     if group == 'libraries':
         subsystem_group = 'main'
@@ -180,25 +223,51 @@ def SAMBA_LIBRARY(bld, libname, source,
     # first create a target for building the object files for this library
     # by separating in this way, we avoid recompiling the C files
     # separately for the install library and the build library
-    bld.SAMBA_SUBSYSTEM(obj_target,
-                        source         = source,
-                        deps           = deps,
-                        public_deps    = public_deps,
-                        includes       = includes,
-                        public_headers = public_headers,
-                        public_headers_install = public_headers_install,
-                        header_path    = header_path,
-                        cflags         = cflags,
-                        group          = subsystem_group,
-                        autoproto      = autoproto,
-                        autoproto_extra_source=autoproto_extra_source,
-                        depends_on     = depends_on,
-                        hide_symbols   = hide_symbols,
-                        allow_warnings = allow_warnings,
-                        pyembed        = pyembed,
-                        pyext          = pyext,
-                        local_include  = local_include,
-                        global_include = global_include)
+    if builtin_target:
+        __t = __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                                        deps=deps,
+                                        public_deps=public_deps,
+                                        includes=includes,
+                                        header_path=header_path,
+                                        builtin_cflags=builtin_cflags,
+                                        builtin_cflags_end=builtin_cflags_end,
+                                        group=group,
+                                        depends_on=depends_on,
+                                        local_include=local_include,
+                                        global_include=global_include,
+                                        allow_warnings=allow_warnings)
+        builtin_subsystem = __t
+    else:
+        builtin_subsystem = None
+    if obj_target:
+        bld.SAMBA_SUBSYSTEM(obj_target,
+                            source         = source,
+                            deps           = deps,
+                            public_deps    = public_deps,
+                            includes       = includes,
+                            public_headers = public_headers,
+                            public_headers_install = public_headers_install,
+                            private_headers= private_headers,
+                            header_path    = header_path,
+                            cflags         = cflags,
+                            cflags_end     = cflags_end,
+                            group          = subsystem_group,
+                            autoproto      = autoproto,
+                            autoproto_extra_source=autoproto_extra_source,
+                            depends_on     = depends_on,
+                            hide_symbols   = hide_symbols,
+                            allow_warnings = allow_warnings,
+                            pyembed        = pyembed,
+                            pyext          = pyext,
+                            local_include  = local_include,
+                            __require_builtin_deps=require_builtin_deps,
+                            global_include = global_include)
+    else:
+        et = bld.SAMBA_SUBSYSTEM(empty_target,
+                                 source=[],
+                                 __force_empty=True,
+                                 __require_builtin_deps=True)
+        et.samba_builtin_subsystem = builtin_subsystem
 
     if BUILTIN_LIBRARY(bld, libname):
         return
@@ -207,9 +276,7 @@ def SAMBA_LIBRARY(bld, libname, source,
         return
 
     # the library itself will depend on that object target
-    deps += ' ' + public_deps
-    deps = TO_LIST(deps)
-    deps.append(obj_target)
+    samba_deps.append(dep_target)
 
     realname = bld.map_shlib_extension(realname, python=(target_type=='PYTHON'))
     link_name = bld.map_shlib_extension(link_name, python=(target_type=='PYTHON'))
@@ -217,15 +284,17 @@ def SAMBA_LIBRARY(bld, libname, source,
     # we don't want any public libraries without version numbers
     if (not private_library and target_type != 'PYTHON' and not realname):
         if vnum is None and soname is None:
-            raise Utils.WafError("public library '%s' must have a vnum" %
+            raise Errors.WafError("public library '%s' must have a vnum" %
                     libname)
-        if pc_files is None and not bld.env['IS_EXTRA_PYTHON']:
-            raise Utils.WafError("public library '%s' must have pkg-config file" %
+        if pc_files is None:
+            raise Errors.WafError("public library '%s' must have pkg-config file" %
                        libname)
-        if public_headers is None and not bld.env['IS_EXTRA_PYTHON']:
-            raise Utils.WafError("public library '%s' must have header files" %
+        if public_headers is None:
+            raise Errors.WafError("public library '%s' must have header files" %
                        libname)
 
+    abi_vnum = vnum
+
     if bundled_name is not None:
         pass
     elif target_type == 'PYTHON' or realname or not private_library:
@@ -235,10 +304,8 @@ def SAMBA_LIBRARY(bld, libname, source,
             bundled_name = libname.replace('_', '-')
     else:
         assert (private_library == True and realname is None)
-        if abi_directory or vnum or soname:
-            bundled_extension=True
-        bundled_name = PRIVATE_NAME(bld, libname.replace('_', '-'),
-                                    bundled_extension, private_library)
+        bundled_name = PRIVATE_NAME(bld, libname.replace('_', '-'))
+        vnum = None
 
     ldflags = TO_LIST(ldflags)
     if bld.env['ENABLE_RELRO'] is True:
@@ -254,10 +321,10 @@ def SAMBA_LIBRARY(bld, libname, source,
         features += ' abi_check'
 
     if pyembed and bld.env['PYTHON_SO_ABI_FLAG']:
-        # For ABI checking, we don't care about the exact Python version.
-        # Replace the Python ABI tag (e.g. ".cpython-35m") by a generic ".py3"
+        # For ABI checking, we don't care about the Python version.
+        # Remove the Python ABI tag (e.g. ".cpython-35m")
         abi_flag = bld.env['PYTHON_SO_ABI_FLAG']
-        replacement = '.py%s' % bld.env['PYTHON_VERSION'].split('.')[0]
+        replacement = ''
         version_libname = libname.replace(abi_flag, replacement)
     else:
         version_libname = libname
@@ -265,26 +332,29 @@ def SAMBA_LIBRARY(bld, libname, source,
     vscript = None
     if bld.env.HAVE_LD_VERSION_SCRIPT:
         if private_library:
-            version = "%s_%s" % (Utils.g_module.APPNAME, Utils.g_module.VERSION)
+            version = bld.env.PRIVATE_VERSION
         elif vnum:
             version = "%s_%s" % (libname, vnum)
         else:
             version = None
         if version:
             vscript = "%s.vscript" % libname
-            bld.ABI_VSCRIPT(version_libname, abi_directory, version, vscript,
-                            abi_match)
-            fullname = apply_pattern(bundled_name, bld.env.shlib_PATTERN)
+            if orig_vscript_map:
+                bld.VSCRIPT_MAP_PRIVATE(version_libname, orig_vscript_map, version, vscript)
+            else:
+                bld.ABI_VSCRIPT(version_libname, abi_directory, version, vscript,
+                                abi_match, private_library)
+            fullname = apply_pattern(bundled_name, bld.env.cshlib_PATTERN)
             fullpath = bld.path.find_or_declare(fullname)
             vscriptpath = bld.path.find_or_declare(vscript)
             if not fullpath:
-                raise Utils.WafError("unable to find fullpath for %s" % fullname)
+                raise Errors.WafError("unable to find fullpath for %s" % fullname)
             if not vscriptpath:
-                raise Utils.WafError("unable to find vscript path for %s" % vscript)
+                raise Errors.WafError("unable to find vscript path for %s" % vscript)
             bld.add_manual_dependency(fullpath, vscriptpath)
-            if Options.is_install:
+            if bld.is_install:
                 # also make the .inst file depend on the vscript
-                instname = apply_pattern(bundled_name + '.inst', bld.env.shlib_PATTERN)
+                instname = apply_pattern(bundled_name + '.inst', bld.env.cshlib_PATTERN)
                 bld.add_manual_dependency(bld.path.find_or_declare(instname), bld.path.find_or_declare(vscript))
             vscript = os.path.join(bld.path.abspath(bld.env), vscript)
 
@@ -295,7 +365,7 @@ def SAMBA_LIBRARY(bld, libname, source,
         target          = bundled_name,
         depends_on      = depends_on,
         samba_ldflags   = ldflags,
-        samba_deps      = deps,
+        samba_deps      = samba_deps,
         samba_includes  = includes,
         version_script  = vscript,
         version_libname = version_libname,
@@ -310,19 +380,27 @@ def SAMBA_LIBRARY(bld, libname, source,
         samba_install   = install,
         abi_directory   = "%s/%s" % (bld.path.abspath(), abi_directory),
         abi_match       = abi_match,
+        abi_vnum        = abi_vnum,
         private_library = private_library,
         grouping_library=grouping_library,
-        allow_undefined_symbols=allow_undefined_symbols
+        allow_undefined_symbols=allow_undefined_symbols,
+        samba_require_builtin_deps=False,
+        samba_builtin_subsystem=builtin_subsystem,
         )
 
     if realname and not link_name:
         link_name = 'shared/%s' % realname
 
     if link_name:
+        if 'waflib.extras.compat15' in sys.modules:
+            link_name = 'default/' + link_name
         t.link_name = link_name
 
     if pc_files is not None and not private_library:
-        bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
+        if pyembed:
+            bld.PKG_CONFIG_FILES(pc_files, vnum=vnum, extra_name=bld.env['PYTHON_SO_ABI_FLAG'])
+        else:
+            bld.PKG_CONFIG_FILES(pc_files, vnum=vnum)
 
     if (manpages is not None and 'XSLTPROC_MANPAGES' in bld.env and
         bld.env['XSLTPROC_MANPAGES']):
@@ -337,10 +415,12 @@ def SAMBA_BINARY(bld, binname, source,
                  deps='',
                  includes='',
                  public_headers=None,
+                 private_headers=None,
                  header_path=None,
                  modules=None,
                  ldflags=None,
                  cflags='',
+                 cflags_end=None,
                  autoproto=None,
                  use_hostcc=False,
                  use_global_deps=True,
@@ -350,18 +430,38 @@ def SAMBA_BINARY(bld, binname, source,
                  local_include=True,
                  global_include=True,
                  subsystem_name=None,
+                 allow_warnings=False,
                  pyembed=False,
                  vars=None,
                  subdir=None,
                  install=True,
                  install_path=None,
-                 enabled=True):
+                 enabled=True,
+                 fuzzer=False,
+                 for_selftest=False):
     '''define a Samba binary'''
 
+    if for_selftest:
+        install=False
+        if not bld.CONFIG_GET('ENABLE_SELFTEST'):
+            enabled=False
+
     if not enabled:
         SET_TARGET_TYPE(bld, binname, 'DISABLED')
         return
 
+    # Fuzzing builds do not build normal binaries
+    # however we must build asn1compile etc
+
+    if not use_hostcc and bld.env.enable_fuzzing != fuzzer:
+        SET_TARGET_TYPE(bld, binname, 'DISABLED')
+        return
+
+    if fuzzer:
+        install = False
+        if ldflags is None:
+            ldflags = bld.env['FUZZ_TARGET_LDFLAGS']
+
     if not SET_TARGET_TYPE(bld, binname, 'BINARY'):
         return
 
@@ -378,14 +478,16 @@ def SAMBA_BINARY(bld, binname, source,
 
     if group == 'binaries':
         subsystem_group = 'main'
+    elif group == 'build_compilers':
+        subsystem_group = 'compiler_libraries'
     else:
         subsystem_group = group
 
     # only specify PIE flags for binaries
-    pie_cflags = cflags
+    pie_cflags = TO_LIST(cflags)
     pie_ldflags = TO_LIST(ldflags)
     if bld.env['ENABLE_PIE'] is True:
-        pie_cflags += ' -fPIE'
+        pie_cflags.extend(TO_LIST('-fPIE'))
         pie_ldflags.extend(TO_LIST('-pie'))
     if bld.env['ENABLE_RELRO'] is True:
         pie_ldflags.extend(TO_LIST('-Wl,-z,relro,-z,now'))
@@ -398,6 +500,7 @@ def SAMBA_BINARY(bld, binname, source,
                         deps           = deps,
                         includes       = includes,
                         cflags         = pie_cflags,
+                        cflags_end     = cflags_end,
                         group          = subsystem_group,
                         autoproto      = autoproto,
                         subsystem_name = subsystem_name,
@@ -405,6 +508,7 @@ def SAMBA_BINARY(bld, binname, source,
                         global_include = global_include,
                         use_hostcc     = use_hostcc,
                         pyext          = pyembed,
+                        allow_warnings = allow_warnings,
                         use_global_deps= use_global_deps)
 
     bld.SET_BUILD_GROUP(group)
@@ -446,6 +550,7 @@ def SAMBA_MODULE(bld, modname, source,
                  autoproto=None,
                  autoproto_extra_source='',
                  cflags='',
+                 cflags_end=None,
                  internal_module=True,
                  local_include=True,
                  global_include=True,
@@ -455,7 +560,8 @@ def SAMBA_MODULE(bld, modname, source,
                  pyembed=False,
                  manpages=None,
                  allow_undefined_symbols=False,
-                 allow_warnings=False
+                 allow_warnings=False,
+                 install=True
                  ):
     '''define a Samba module.'''
 
@@ -475,6 +581,7 @@ def SAMBA_MODULE(bld, modname, source,
                     autoproto=autoproto,
                     autoproto_extra_source=autoproto_extra_source,
                     cflags=cflags,
+                    cflags_end=cflags_end,
                     local_include=local_include,
                     global_include=global_include,
                     allow_warnings=allow_warnings,
@@ -514,6 +621,7 @@ def SAMBA_MODULE(bld, modname, source,
                       deps=deps,
                       includes=includes,
                       cflags=cflags,
+                      cflags_end=cflags_end,
                       realname = realname,
                       autoproto = autoproto,
                       local_include=local_include,
@@ -525,24 +633,135 @@ def SAMBA_MODULE(bld, modname, source,
                       pyembed=pyembed,
                       manpages=manpages,
                       allow_undefined_symbols=allow_undefined_symbols,
-                      allow_warnings=allow_warnings
+                      allow_warnings=allow_warnings,
+                      private_library=True,
+                      install=install
                       )
 
 
 Build.BuildContext.SAMBA_MODULE = SAMBA_MODULE
 
+#################################################################
+def SAMBA_PLUGIN(bld, pluginname, source,
+                 deps='',
+                 includes='',
+                 vnum=None,
+                 soname=None,
+                 cflags='',
+                 ldflags='',
+                 local_include=True,
+                 global_include=True,
+                 vars=None,
+                 subdir=None,
+                 realname=None,
+                 keep_underscore=False,
+                 autoproto=None,
+                 autoproto_extra_source='',
+                 install_path=None,
+                 install=True,
+                 manpages=None,
+                 require_builtin_deps=True,
+                 allow_undefined_symbols=False,
+                 enabled=True):
+    '''define an external plugin.'''
+
+    bld.ASSERT(realname, "You must specify a realname for SAMBA_PLUGIN(%s)" % pluginname)
+
+    source = bld.EXPAND_VARIABLES(source, vars=vars)
+    if subdir:
+        source = bld.SUBDIR(subdir, source)
+
+    build_name = "_plugin_%s" % (pluginname)
+    build_link_name = "plugins/%s" % (realname)
+
+    bld.SAMBA_LIBRARY(pluginname,
+                      source,
+                      bundled_name=build_name,
+                      link_name=build_link_name,
+                      target_type='PLUGIN',
+                      deps=deps,
+                      includes=includes,
+                      vnum=vnum,
+                      soname=soname,
+                      cflags=cflags,
+                      ldflags=ldflags,
+                      realname=realname,
+                      autoproto=autoproto,
+                      autoproto_extra_source=autoproto_extra_source,
+                      local_include=local_include,
+                      global_include=global_include,
+                      vars=vars,
+                      group='main',
+                      install_path=install_path,
+                      install=install,
+                      manpages=manpages,
+                      require_builtin_deps=require_builtin_deps,
+                      builtin_cflags=cflags,
+                      hide_symbols=True,
+                      public_headers=[],
+                      public_headers_install=False,
+                      pc_files=[],
+                      allow_undefined_symbols=allow_undefined_symbols,
+                      allow_warnings=False,
+                      enabled=enabled)
+Build.BuildContext.SAMBA_PLUGIN = SAMBA_PLUGIN
+
+def __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                              deps='',
+                              public_deps='',
+                              includes='',
+                              public_headers=None,
+                              public_headers_install=True,
+                              private_headers=None,
+                              header_path=None,
+                              builtin_cflags='',
+                              builtin_cflags_end=None,
+                              group='main',
+                              autoproto=None,
+                              autoproto_extra_source='',
+                              depends_on='',
+                              local_include=True,
+                              global_include=True,
+                              allow_warnings=False):
+
+    bld.ASSERT(builtin_target.endswith('.builtin.objlist'),
+               "builtin_target[%s] does not end with '.builtin.objlist'" %
+               (builtin_target))
+    return bld.SAMBA_SUBSYSTEM(builtin_target, source,
+                               deps=deps,
+                               public_deps=public_deps,
+                               includes=includes,
+                               public_headers=public_headers,
+                               public_headers_install=public_headers_install,
+                               private_headers=private_headers,
+                               header_path=header_path,
+                               cflags=builtin_cflags,
+                               cflags_end=builtin_cflags_end,
+                               hide_symbols=True,
+                               group=group,
+                               target_type='BUILTIN',
+                               autoproto=autoproto,
+                               autoproto_extra_source=autoproto_extra_source,
+                               depends_on=depends_on,
+                               local_include=local_include,
+                               global_include=global_include,
+                               allow_warnings=allow_warnings,
+                               __require_builtin_deps=True)
 
 #################################################################
 def SAMBA_SUBSYSTEM(bld, modname, source,
                     deps='',
                     public_deps='',
+                    __force_empty=False,
                     includes='',
                     public_headers=None,
                     public_headers_install=True,
+                    private_headers=None,
                     header_path=None,
                     cflags='',
                     cflags_end=None,
                     group='main',
+                    target_type='SUBSYSTEM',
                     init_function_sentinel=None,
                     autoproto=None,
                     autoproto_extra_source='',
@@ -557,18 +776,28 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                     vars=None,
                     subdir=None,
                     hide_symbols=False,
+                    __require_builtin_deps=False,
+                    provide_builtin_linking=False,
+                    builtin_cflags='',
                     allow_warnings=False,
                     pyext=False,
                     pyembed=False):
     '''define a Samba subsystem'''
 
+    # We support:
+    # - SUBSYSTEM: a normal subsystem from SAMBA_SUBSYSTEM()
+    # - BUILTIN:   a hidden subsystem from __SAMBA_SUBSYSTEM_BUILTIN()
+    if target_type not in ['SUBSYSTEM', 'BUILTIN']:
+        raise Errors.WafError("target_type[%s] not supported in SAMBA_SUBSYSTEM('%s')" %
+                              (target_type, modname))
+
     if not enabled:
         SET_TARGET_TYPE(bld, modname, 'DISABLED')
         return
 
     # remember empty subsystems, so we can strip the dependencies
     if ((source == '') or (source == [])):
-        if deps == '' and public_deps == '':
+        if not __force_empty and deps == '' and public_deps == '':
             SET_TARGET_TYPE(bld, modname, 'EMPTY')
             return
         empty_c = modname + '.empty.c'
@@ -577,7 +806,7 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
                             target=empty_c)
         source=empty_c
 
-    if not SET_TARGET_TYPE(bld, modname, 'SUBSYSTEM'):
+    if not SET_TARGET_TYPE(bld, modname, target_type):
         return
 
     source = bld.EXPAND_VARIABLES(source, vars=vars)
@@ -601,6 +830,7 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
         target         = modname,
         samba_cflags   = CURRENT_CFLAGS(bld, modname, cflags,
                                         allow_warnings=allow_warnings,
+                                        use_hostcc=use_hostcc,
                                         hide_symbols=hide_symbols),
         depends_on     = depends_on,
         samba_deps     = TO_LIST(deps),
@@ -611,6 +841,8 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
         samba_subsystem= subsystem_name,
         samba_use_hostcc = use_hostcc,
         samba_use_global_deps = use_global_deps,
+        samba_require_builtin_deps = __require_builtin_deps,
+        samba_builtin_subsystem = None,
         )
 
     if cflags_end is not None:
@@ -621,6 +853,39 @@ def SAMBA_SUBSYSTEM(bld, modname, source,
     if public_headers is not None:
         bld.PUBLIC_HEADERS(public_headers, header_path=header_path,
                            public_headers_install=public_headers_install)
+
+    if provide_builtin_linking:
+
+        if use_hostcc:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with use_hostcc=True" %
+                                  modname)
+
+        if pyext or pyembed:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with pyext=True nor pyembed=True" %
+                                  modname)
+
+        if __require_builtin_deps:
+            raise Errors.WafError("subsystem[%s] provide_builtin_linking=True " +
+                                  "not allowed with __require_builtin_deps=True" %
+                                  modname)
+
+        builtin_target = modname + '.builtin.objlist'
+        tbuiltin = __SAMBA_SUBSYSTEM_BUILTIN(bld, builtin_target, source,
+                                             deps=deps,
+                                             public_deps=public_deps,
+                                             includes=includes,
+                                             header_path=header_path,
+                                             builtin_cflags=builtin_cflags,
+                                             builtin_cflags_end='-D_PUBLIC_=_PRIVATE_',
+                                             group=group,
+                                             depends_on=depends_on,
+                                             local_include=local_include,
+                                             global_include=global_include,
+                                             allow_warnings=allow_warnings)
+        t.samba_builtin_subsystem = tbuiltin
+
     return t
 
 
@@ -631,6 +896,7 @@ def SAMBA_GENERATOR(bld, name, rule, source='', target='',
                     group='generators', enabled=True,
                     public_headers=None,
                     public_headers_install=True,
+                    private_headers=None,
                     header_path=None,
                     vars=None,
                     dep_vars=[],
@@ -643,6 +909,7 @@ def SAMBA_GENERATOR(bld, name, rule, source='', target='',
     if not enabled:
         return
 
+    dep_vars = TO_LIST(dep_vars)
     dep_vars.append('ruledeps')
     dep_vars.append('SAMBA_GENERATOR_VARS')
 
@@ -653,7 +920,7 @@ def SAMBA_GENERATOR(bld, name, rule, source='', target='',
         target=target,
         shell=isinstance(rule, str),
         update_outputs=True,
-        before='cc',
+        before='c',
         ext_out='.c',
         samba_type='GENERATOR',
         dep_vars = dep_vars,
@@ -681,13 +948,13 @@ def SETUP_BUILD_GROUPS(bld):
     bld.p_ln = bld.srcnode # we do want to see all targets!
     bld.env['USING_BUILD_GROUPS'] = True
     bld.add_group('setup')
-    bld.add_group('build_compiler_source')
+    bld.add_group('generators')
+    bld.add_group('hostcc_base_build_source')
+    bld.add_group('hostcc_base_build_main')
+    bld.add_group('hostcc_build_source')
+    bld.add_group('hostcc_build_main')
     bld.add_group('vscripts')
     bld.add_group('base_libraries')
-    bld.add_group('generators')
-    bld.add_group('compiler_prototypes')
-    bld.add_group('compiler_libraries')
-    bld.add_group('build_compilers')
     bld.add_group('build_source')
     bld.add_group('prototypes')
     bld.add_group('headers')
@@ -707,27 +974,11 @@ Build.BuildContext.SET_BUILD_GROUP = SET_BUILD_GROUP
 
 
 
-@conf
-def ENABLE_TIMESTAMP_DEPENDENCIES(conf):
-    """use timestamps instead of file contents for deps
-    this currently doesn't work"""
-    def h_file(filename):
-        import stat
-        st = os.stat(filename)
-        if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
-        m = Utils.md5()
-        m.update(str(st.st_mtime))
-        m.update(str(st.st_size))
-        m.update(filename)
-        return m.digest()
-    Utils.h_file = h_file
-
-
 def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
     '''used to copy scripts from the source tree into the build directory
        for use by selftest'''
 
-    source = bld.path.ant_glob(pattern)
+    source = bld.path.ant_glob(pattern, flat=True)
 
     bld.SET_BUILD_GROUP('build_source')
     for s in TO_LIST(source):
@@ -737,14 +988,14 @@ def SAMBA_SCRIPT(bld, name, pattern, installdir, installname=None):
         target = os.path.join(installdir, iname)
         tgtdir = os.path.dirname(os.path.join(bld.srcnode.abspath(bld.env), '..', target))
         mkdir_p(tgtdir)
-        link_src = os.path.normpath(os.path.join(bld.curdir, s))
+        link_src = os.path.normpath(os.path.join(bld.path.abspath(), s))
         link_dst = os.path.join(tgtdir, os.path.basename(iname))
         if os.path.islink(link_dst) and os.readlink(link_dst) == link_src:
             continue
-        if os.path.exists(link_dst):
+        if os.path.islink(link_dst):
             os.unlink(link_dst)
         Logs.info("symlink: %s -> %s/%s" % (s, installdir, iname))
-        os.symlink(link_src, link_dst)
+        symlink(link_src, link_dst)
 Build.BuildContext.SAMBA_SCRIPT = SAMBA_SCRIPT
 
 
@@ -758,10 +1009,10 @@ def copy_and_fix_python_path(task):
         replacement="""sys.path.insert(0, "%s")
 sys.path.insert(1, "%s")""" % (task.env["PYTHONARCHDIR"], task.env["PYTHONDIR"])
 
-    if task.env["PYTHON"][0] == "/":
-        replacement_shebang = "#!%s\n" % task.env["PYTHON"]
+    if task.env["PYTHON"][0].startswith("/"):
+        replacement_shebang = "#!%s\n" % task.env["PYTHON"][0]
     else:
-        replacement_shebang = "#!/usr/bin/env %s\n" % task.env["PYTHON"]
+        replacement_shebang = "#!/usr/bin/env %s\n" % task.env["PYTHON"][0]
 
     installed_location=task.outputs[0].bldpath(task.env)
     source_file = open(task.inputs[0].srcpath(task.env))
@@ -769,7 +1020,7 @@ sys.path.insert(1, "%s")""" % (task.env["PYTHONARCHDIR"], task.env["PYTHONDIR"])
     lineno = 0
     for line in source_file:
         newline = line
-        if (lineno == 0 and task.env["PYTHON_SPECIFIED"] is True and
+        if (lineno == 0 and
                 line[:2] == "#!"):
             newline = replacement_shebang
         elif pattern in line:
@@ -777,7 +1028,7 @@ sys.path.insert(1, "%s")""" % (task.env["PYTHONARCHDIR"], task.env["PYTHONDIR"])
         installed_file.write(newline)
         lineno = lineno + 1
     installed_file.close()
-    os.chmod(installed_location, 0755)
+    os.chmod(installed_location, 0o755)
     return 0
 
 def copy_and_fix_perl_path(task):
@@ -805,7 +1056,7 @@ def copy_and_fix_perl_path(task):
         installed_file.write(newline)
         lineno = lineno + 1
     installed_file.close()
-    os.chmod(installed_location, 0755)
+    os.chmod(installed_location, 0o755)
     return 0
 
 
@@ -813,6 +1064,8 @@ def install_file(bld, destdir, file, chmod=MODE_644, flat=False,
                  python_fixup=False, perl_fixup=False,
                  destname=None, base_name=None):
     '''install a file'''
+    if not isinstance(file, str):
+        file = file.abspath()
     destdir = bld.EXPAND_VARIABLES(destdir)
     if not destname:
         destname = file
@@ -856,11 +1109,11 @@ Build.BuildContext.INSTALL_FILES = INSTALL_FILES
 def INSTALL_WILDCARD(bld, destdir, pattern, chmod=MODE_644, flat=False,
                      python_fixup=False, exclude=None, trim_path=None):
     '''install a set of files matching a wildcard pattern'''
-    files=TO_LIST(bld.path.ant_glob(pattern))
+    files=TO_LIST(bld.path.ant_glob(pattern, flat=True))
     if trim_path:
         files2 = []
         for f in files:
-            files2.append(os_path_relpath(f, trim_path))
+            files2.append(os.path.relpath(f, trim_path))
         files = files2
 
     if exclude:
@@ -871,13 +1124,33 @@ def INSTALL_WILDCARD(bld, destdir, pattern, chmod=MODE_644, flat=False,
                   python_fixup=python_fixup, base_name=trim_path)
 Build.BuildContext.INSTALL_WILDCARD = INSTALL_WILDCARD
 
+def INSTALL_DIR(bld, path, chmod=0o755, env=None):
+    """Install a directory if it doesn't exist, always set permissions."""
+
+    if not path:
+        return []
+
+    destpath = bld.EXPAND_VARIABLES(path)
+    if Options.options.destdir:
+        destpath = os.path.join(Options.options.destdir, destpath.lstrip(os.sep))
 
-def INSTALL_DIRS(bld, destdir, dirs):
+    if bld.is_install > 0:
+        if not os.path.isdir(destpath):
+            try:
+                Logs.info('* create %s', destpath)
+                os.makedirs(destpath)
+                os.chmod(destpath, chmod)
+            except OSError as e:
+                if not os.path.isdir(destpath):
+                    raise Errors.WafError("Cannot create the folder '%s' (error: %s)" % (path, e))
+Build.BuildContext.INSTALL_DIR = INSTALL_DIR
+
+def INSTALL_DIRS(bld, destdir, dirs, chmod=0o755, env=None):
     '''install a set of directories'''
     destdir = bld.EXPAND_VARIABLES(destdir)
     dirs = bld.EXPAND_VARIABLES(dirs)
     for d in TO_LIST(dirs):
-        bld.install_dir(os.path.join(destdir, d))
+        INSTALL_DIR(bld, os.path.join(destdir, d), chmod, env)
 Build.BuildContext.INSTALL_DIRS = INSTALL_DIRS
 
 
@@ -900,13 +1173,17 @@ def SAMBAMANPAGES(bld, manpages, extra_source=None):
     '''build and install manual pages'''
     bld.env.SAMBA_EXPAND_XSL = bld.srcnode.abspath() + '/docs-xml/xslt/expand-sambadoc.xsl'
     bld.env.SAMBA_MAN_XSL = bld.srcnode.abspath() + '/docs-xml/xslt/man.xsl'
-    bld.env.SAMBA_CATALOG = bld.srcnode.abspath() + '/bin/default/docs-xml/build/catalog.xml'
+    bld.env.SAMBA_CATALOG = bld.bldnode.abspath() + '/docs-xml/build/catalog.xml'
     bld.env.SAMBA_CATALOGS = 'file:///etc/xml/catalog file:///usr/local/share/xml/catalog file://' + bld.env.SAMBA_CATALOG
 
     for m in manpages.split():
-        source = m + '.xml'
+        source = [m + '.xml']
         if extra_source is not None:
             source = [source, extra_source]
+        # ${SRC[1]} and ${SRC[2]} are not referenced in the
+        # SAMBA_GENERATOR but trigger the dependency calculation so
+        # ensures that manpages are rebuilt when these change.
+        source += ['build/DTD/samba.entities', 'build/DTD/samba.build.version']
         bld.SAMBA_GENERATOR(m,
                             source=source,
                             target=m,
@@ -920,59 +1197,6 @@ def SAMBAMANPAGES(bld, manpages, extra_source=None):
         bld.INSTALL_FILES('${MANDIR}/man%s' % m[-1], m, flat=True)
 Build.BuildContext.SAMBAMANPAGES = SAMBAMANPAGES
 
-#############################################################
-# give a nicer display when building different types of files
-def progress_display(self, msg, fname):
-    col1 = Logs.colors(self.color)
-    col2 = Logs.colors.NORMAL
-    total = self.position[1]
-    n = len(str(total))
-    fs = '[%%%dd/%%%dd] %s %%s%%s%%s\n' % (n, n, msg)
-    return fs % (self.position[0], self.position[1], col1, fname, col2)
-
-def link_display(self):
-    if Options.options.progress_bar != 0:
-        return Task.Task.old_display(self)
-    fname = self.outputs[0].bldpath(self.env)
-    return progress_display(self, 'Linking', fname)
-Task.TaskBase.classes['cc_link'].display = link_display
-
-def samba_display(self):
-    if Options.options.progress_bar != 0:
-        return Task.Task.old_display(self)
-
-    targets    = LOCAL_CACHE(self, 'TARGET_TYPE')
-    if self.name in targets:
-        target_type = targets[self.name]
-        type_map = { 'GENERATOR' : 'Generating',
-                     'PROTOTYPE' : 'Generating'
-                     }
-        if target_type in type_map:
-            return progress_display(self, type_map[target_type], self.name)
-
-    if len(self.inputs) == 0:
-        return Task.Task.old_display(self)
-
-    fname = self.inputs[0].bldpath(self.env)
-    if fname[0:3] == '../':
-        fname = fname[3:]
-    ext_loc = fname.rfind('.')
-    if ext_loc == -1:
-        return Task.Task.old_display(self)
-    ext = fname[ext_loc:]
-
-    ext_map = { '.idl' : 'Compiling IDL',
-                '.et'  : 'Compiling ERRTABLE',
-                '.asn1': 'Compiling ASN1',
-                '.c'   : 'Compiling' }
-    if ext in ext_map:
-        return progress_display(self, ext_map[ext], fname)
-    return Task.Task.old_display(self)
-
-Task.TaskBase.classes['Task'].old_display = Task.TaskBase.classes['Task'].display
-Task.TaskBase.classes['Task'].display = samba_display
-
-
 @after('apply_link')
 @feature('cshlib')
 def apply_bundle_remove_dynamiclib_patch(self):