python: Change except: statement to except Exception:
[sfrench/samba-autobuild/.git] / source4 / scripting / bin / upgradeprovision
index 1a684bd6756fafc674e5c0ea6e523857d426d5e7..aec0774a9d2046bc2e7d263110c3157246300aba 100755 (executable)
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: expandtab
 #
-# Copyright (C) Matthieu Patou <mat@matws.net> 2009
+# Copyright (C) Matthieu Patou <mat@matws.net> 2009 - 2010
 #
 # Based on provision a Samba4 server by
 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007-2008
@@ -33,28 +33,36 @@ import traceback
 # Allow to run from s4 source directory (without installing samba)
 sys.path.insert(0, "bin/python")
 
+import ldb
 import samba
 import samba.getopt as options
+
+from base64 import b64encode
 from samba.credentials import DONT_USE_KERBEROS
 from samba.auth import system_session, admin_session
-from samba import Ldb, version
-from ldb import SCOPE_ONELEVEL, SCOPE_SUBTREE, SCOPE_BASE,\
-                FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE,\
-                MessageElement, Message, Dn
-from samba import param
-from samba.misc import messageEltFlagToString
-from samba.provision import find_setup_dir, get_domain_descriptor,\
-                            get_config_descriptor, secretsdb_self_join,\
-                            set_gpo_acl, getpolicypath,create_gpo_struct,\
-                            ProvisioningError, getLastProvisionUSN,\
-                            get_max_usn, updateProvisionUSN
+from ldb import (SCOPE_SUBTREE, SCOPE_BASE,
+                FLAG_MOD_REPLACE, FLAG_MOD_ADD, FLAG_MOD_DELETE,
+                MessageElement, Message, Dn, LdbError)
+from samba import param, dsdb, Ldb
+from samba.common import confirm
+from samba.provision import (get_domain_descriptor, find_provision_key_parameters,
+                            get_config_descriptor,
+                            ProvisioningError, get_last_provision_usn,
+                            get_max_usn, update_provision_usn, setup_path)
 from samba.schema import get_linked_attributes, Schema, get_schema_descriptor
-from samba.dcerpc import security, drsblobs
+from samba.dcerpc import security, drsblobs, xattr
 from samba.ndr import ndr_unpack
-from samba.dcerpc.misc import SEC_CHAN_BDC
-from samba.upgradehelpers import dn_sort, get_paths, newprovision,\
-                                 find_provision_key_parameters, get_ldbs,\
-                                 usn_in_range, identic_rename, get_diff_sddls
+from samba.upgradehelpers import (dn_sort, get_paths, newprovision,
+                                 get_ldbs,
+                                 usn_in_range, identic_rename, get_diff_sddls,
+                                 update_secrets, CHANGE, ERROR, SIMPLE,
+                                 CHANGEALL, GUESS, CHANGESD, PROVISION,
+                                 updateOEMInfo, getOEMInfo, update_gpo,
+                                 delta_update_basesamdb, update_policyids,
+                                 update_machine_account_password,
+                                 search_constructed_attrs_stored,
+                                 int64range2str, update_dns_account_password,
+                                 increment_calculated_keyversion_number)
 
 replace=2**FLAG_MOD_REPLACE
 add=2**FLAG_MOD_ADD
@@ -66,13 +74,6 @@ never=0
 # somehow ...
 
 #Errors are always logged
-ERROR =     -1
-SIMPLE =     0x00
-CHANGE =     0x01
-CHANGESD =     0x02
-GUESS =     0x04
-PROVISION =    0x08
-CHANGEALL =    0xff
 
 __docformat__ = "restructuredText"
 
@@ -81,20 +82,24 @@ __docformat__ = "restructuredText"
 # This is most probably because they are populated automatcally when object is
 # created
 # This also apply to imported object from reference provision
-hashAttrNotCopied = {   "dn": 1, "whenCreated": 1, "whenChanged": 1,
-                        "objectGUID": 1, "uSNCreated": 1,
-                        "replPropertyMetaData": 1, "uSNChanged": 1,
-                        "parentGUID": 1, "objectCategory": 1,
-                        "distinguishedName": 1, "nTMixedDomain": 1,
-                        "showInAdvancedViewOnly": 1, "instanceType": 1,
-                        "msDS-Behavior-Version":1, "nextRid":1, "cn": 1,
-                        "versionNumber":1, "lmPwdHistory":1, "pwdLastSet": 1,
-                        "ntPwdHistory":1, "unicodePwd":1,"dBCSPwd":1,
-                        "supplementalCredentials":1, "gPCUserExtensionNames":1,
-                        "gPCMachineExtensionNames":1,"maxPwdAge":1, "secret":1,
-                        "possibleInferiors":1, "privilege":1,
-                        "sAMAccountType":1 }
+replAttrNotCopied = [   "dn", "whenCreated", "whenChanged", "objectGUID",
+                        "parentGUID", "objectCategory", "distinguishedName",
+                        "instanceType", "cn",
+                        "lmPwdHistory", "pwdLastSet", "ntPwdHistory",
+                        "unicodePwd", "dBCSPwd", "supplementalCredentials",
+                        "gPCUserExtensionNames", "gPCMachineExtensionNames",
+                        "maxPwdAge", "secret", "possibleInferiors", "privilege",
+                        "sAMAccountType", "oEMInformation", "creationTime" ]
+
+nonreplAttrNotCopied = ["uSNCreated", "replPropertyMetaData", "uSNChanged",
+                        "nextRid" ,"rIDNextRID", "rIDPreviousAllocationPool"]
+
+nonDSDBAttrNotCopied = ["msDS-KeyVersionNumber", "priorSecret", "priorWhenChanged"]
+
 
+attrNotCopied = replAttrNotCopied
+attrNotCopied.extend(nonreplAttrNotCopied)
+attrNotCopied.extend(nonDSDBAttrNotCopied)
 # Usually for an object that already exists we do not overwrite attributes as
 # they might have been changed for good reasons. Anyway for a few of them it's
 # mandatory to replace them otherwise the provision will be broken somehow.
@@ -108,12 +113,19 @@ hashOverwrittenAtt = {  "prefixMap": replace, "systemMayContain": replace,
                         "wellKnownObjects":replace, "privilege":never,
                         "defaultSecurityDescriptor": replace,
                         "rIDAvailablePool": never,
-                        "defaultSecurityDescriptor": replace + add }
-
-
+                        "versionNumber" : add,
+                        "rIDNextRID": add, "rIDUsedPool": never,
+                        "defaultSecurityDescriptor": replace + add,
+                        "isMemberOfPartialAttributeSet": delete,
+                        "attributeDisplayNames": replace + add,
+                        "versionNumber": add}
+
+dnNotToRecalculate = []
+dnToRecalculate = []
 backlinked = []
-forwardlinked = {}
+forwardlinked = set()
 dn_syntax_att = []
+not_replicated = []
 def define_what_to_log(opts):
     what = 0
     if opts.debugchange:
@@ -139,13 +151,17 @@ parser.add_option("--setupdir", type="string", metavar="DIR",
                   help="directory with setup files")
 parser.add_option("--debugprovision", help="Debug provision", action="store_true")
 parser.add_option("--debugguess", action="store_true",
-                  help="Print information on what is different but won't be changed")
+                  help="Print information on which values are guessed")
 parser.add_option("--debugchange", action="store_true",
                   help="Print information on what is different but won't be changed")
 parser.add_option("--debugchangesd", action="store_true",
-                  help="Print information security descriptors differences")
+                  help="Print security descriptor differences")
 parser.add_option("--debugall", action="store_true",
                   help="Print all available information (very verbose)")
+parser.add_option("--resetfileacl", action="store_true",
+                  help="Force a reset on filesystem acls in sysvol / netlogon share")
+parser.add_option("--fixntacl", action="store_true",
+                  help="Only fix NT ACLs in sysvol / netlogon share")
 parser.add_option("--full", action="store_true",
                   help="Perform full upgrade of the samdb (schema, configuration, new objects, ...")
 
@@ -153,6 +169,8 @@ opts = parser.parse_args()[0]
 
 handler = logging.StreamHandler(sys.stdout)
 upgrade_logger = logging.getLogger("upgradeprovision")
+upgrade_logger.setLevel(logging.INFO)
+
 upgrade_logger.addHandler(handler)
 
 provision_logger = logging.getLogger("provision")
@@ -175,9 +193,6 @@ smbconf = lp.configfile
 
 creds = credopts.get_credentials(lp)
 creds.set_kerberos_state(DONT_USE_KERBEROS)
-setup_dir = opts.setupdir
-if setup_dir is None:
-    setup_dir = find_setup_dir()
 
 
 
@@ -190,6 +205,7 @@ def check_for_DNS(refprivate, private):
                     provision"""
 
     spnfile = "%s/spn_update_list" % private
+    dnsfile = "%s/dns_update_list" % private
     namedfile = lp.get("dnsupdate:path")
 
     if not namedfile:
@@ -198,6 +214,9 @@ def check_for_DNS(refprivate, private):
     if not os.path.exists(spnfile):
         shutil.copy("%s/spn_update_list" % refprivate, "%s" % spnfile)
 
+    if not os.path.exists(dnsfile):
+        shutil.copy("%s/dns_update_list" % refprivate, "%s" % dnsfile)
+
     destdir = "%s/new_dns" % private
     dnsdir = "%s/dns" % private
 
@@ -208,11 +227,11 @@ def check_for_DNS(refprivate, private):
             os.mkdir(dnsdir)
         shutil.copy("%s/named.conf" % refprivate, "%s/named.conf" % destdir)
         shutil.copy("%s/named.txt" % refprivate, "%s/named.txt" % destdir)
-        message(SIMPLE, "It seems that you provision didn't integrate new rules "
-                "for dynamic dns update of domain related entries")
+        message(SIMPLE, "It seems that your provision did not integrate "
+                "new rules for dynamic dns update of domain related entries")
         message(SIMPLE, "A copy of the new bind configuration files and "
-                "template as been put in %s, you should read them and configure dynamic "
-                " dns update" % destdir)
+                "template has been put in %s, you should read them and "
+                "configure dynamic dns updates" % destdir)
 
 
 def populate_links(samdb, schemadn):
@@ -226,7 +245,28 @@ def populate_links(samdb, schemadn):
     linkedAttHash = get_linked_attributes(Dn(samdb, str(schemadn)), samdb)
     backlinked.extend(linkedAttHash.values())
     for t in linkedAttHash.keys():
-        forwardlinked[t] = 1
+        forwardlinked.add(t)
+
+def isReplicated(att):
+    """ Indicate if the attribute is replicated or not
+
+    :param att: Name of the attribute to be tested
+    :return: True is the attribute is replicated, False otherwise
+    """
+
+    return (att not in not_replicated)
+
+def populateNotReplicated(samdb, schemadn):
+    """Populate an array with all the attributes that are not replicated
+
+    :param samdb: A LDB object for sam.ldb file
+    :param schemadn: DN of the schema for the partition"""
+    res = samdb.search(expression="(&(objectclass=attributeSchema)(systemflags:1.2.840.113556.1.4.803:=1))", base=Dn(samdb,
+                        str(schemadn)), scope=SCOPE_SUBTREE,
+                        attrs=["lDAPDisplayName"])
+    for elem in res:
+        not_replicated.append(str(elem["lDAPDisplayName"]))
+
 
 def populate_dnsyntax(samdb, schemadn):
     """Populate an array with all the attributes that have DN synthax
@@ -251,12 +291,13 @@ def sanitychecks(samdb, names):
                          scope=SCOPE_SUBTREE, attrs=["dn"],
                          controls=["search_options:1:2"])
     if len(res) == 0:
-        print "No DC found, your provision is most probably hardly broken !"
+        print "No DC found. Your provision is most probably broken!"
         return False
     elif len(res) != 1:
-        print "Found %d domain controllers, for the moment upgradeprovision" \
-              "is not able to handle upgrade on domain with more than one DC, please demote" \
-              " the other(s) DC(s) before upgrading" % len(res)
+        print "Found %d domain controllers. For the moment " \
+              "upgradeprovision is not able to handle an upgrade on a " \
+              "domain with more than one DC. Please demote the other " \
+              "DC(s) before upgrading" % len(res)
         return False
     else:
         return True
@@ -285,7 +326,7 @@ def print_provision_key_parameters(names):
     message(GUESS, "domainlevel :" + str(names.domainlevel))
 
 
-def handle_special_case(att, delta, new, old, usn):
+def handle_special_case(att, delta, new, old, useReplMetadata, basedn, aldb):
     """Define more complicate update rules for some attributes
 
     :param att: The attribute to be updated
@@ -293,25 +334,45 @@ def handle_special_case(att, delta, new, old, usn):
                   between the updated object and the reference one
     :param new: The reference object
     :param old: The Updated object
-    :param usn: The highest usn modified by a previous (upgrade)provision
+    :param useReplMetadata: A boolean that indicate if the update process
+                use replPropertyMetaData to decide what has to be updated.
+    :param basedn: The base DN of the provision
+    :param aldb: An ldb object used to build DN
     :return: True to indicate that the attribute should be kept, False for
              discarding it"""
 
-    flag = delta.get(att).flags()
     # We do most of the special case handle if we do not have the
     # highest usn as otherwise the replPropertyMetaData will guide us more
     # correctly
-    if usn == None:
+    if not useReplMetadata:
+        flag = delta.get(att).flags()
+        if (att == "sPNMappings" and flag == FLAG_MOD_REPLACE and
+            ldb.Dn(aldb, "CN=Directory Service,CN=Windows NT,"
+                        "CN=Services,CN=Configuration,%s" % basedn)
+                        == old[0].dn):
+            return True
+        if (att == "userAccountControl" and flag == FLAG_MOD_REPLACE and
+            ldb.Dn(aldb, "CN=Administrator,CN=Users,%s" % basedn)
+                        == old[0].dn):
+            message(SIMPLE, "We suggest that you change the userAccountControl"
+                            " for user Administrator from value %d to %d" %
+                            (int(str(old[0][att])), int(str(new[0][att]))))
+            return False
+        if (att == "minPwdAge" and flag == FLAG_MOD_REPLACE):
+            if (long(str(old[0][att])) == 0):
+                delta[att] = MessageElement(new[0][att], FLAG_MOD_REPLACE, att)
+            return True
+
         if (att == "member" and flag == FLAG_MOD_REPLACE):
             hash = {}
             newval = []
             changeDelta=0
             for elem in old[0][att]:
-                hash[str(elem)]=1
+                hash[str(elem).lower()]=1
                 newval.append(str(elem))
 
             for elem in new[0][att]:
-                if not hash.has_key(str(elem)):
+                if not hash.has_key(str(elem).lower()):
                     changeDelta=1
                     newval.append(str(elem))
             if changeDelta == 1:
@@ -320,9 +381,9 @@ def handle_special_case(att, delta, new, old, usn):
                 delta.remove(att)
             return True
 
-        if (att == "gPLink" or att == "gPCFileSysPath") and \
-            flag == FLAG_MOD_REPLACE and\
-            str(new[0].dn).lower() == str(old[0].dn).lower():
+        if (att in ("gPLink", "gPCFileSysPath") and
+            flag == FLAG_MOD_REPLACE and
+            str(new[0].dn).lower() == str(old[0].dn).lower()):
             delta.remove(att)
             return True
 
@@ -333,10 +394,10 @@ def handle_special_case(att, delta, new, old, usn):
             ref == old and ref == abs(new)
             return True
 
-        if (att == "adminDisplayName" or att == "adminDescription"):
+        if att in ("adminDisplayName", "adminDescription"):
             return True
 
-        if (str(old[0].dn) == "CN=Samba4-Local-Domain, %s" % (str(names.schemadn))\
+        if (str(old[0].dn) == "CN=Samba4-Local-Domain, %s" % (names.schemadn)
             and att == "defaultObjectCategory" and flag == FLAG_MOD_REPLACE):
             return True
 
@@ -347,24 +408,27 @@ def handle_special_case(att, delta, new, old, usn):
         if (str(old[0].dn) == "%s" % (str(names.rootdn))
                 and att == "subRefs" and flag == FLAG_MOD_REPLACE):
             return True
-
+        #Allow to change revision of ForestUpdates objects
+        if (att == "revision" or att == "objectVersion"):
+            if str(delta.dn).lower().find("domainupdates") and str(delta.dn).lower().find("forestupdates") > 0:
+                return True
         if str(delta.dn).endswith("CN=DisplaySpecifiers, %s" % names.configdn):
             return True
 
     # This is a bit of special animal as we might have added
     # already SPN entries to the list that has to be modified
     # So we go in detail to try to find out what has to be added ...
-    if ( att == "servicePrincipalName" and flag == FLAG_MOD_REPLACE):
+    if (att == "servicePrincipalName" and delta.get(att).flags() == FLAG_MOD_REPLACE):
         hash = {}
         newval = []
-        changeDelta=0
+        changeDelta = 0
         for elem in old[0][att]:
             hash[str(elem)]=1
             newval.append(str(elem))
 
         for elem in new[0][att]:
             if not hash.has_key(str(elem)):
-                changeDelta=1
+                changeDelta = 1
                 newval.append(str(elem))
         if changeDelta == 1:
             delta[att] = MessageElement(newval, FLAG_MOD_REPLACE, att)
@@ -374,88 +438,8 @@ def handle_special_case(att, delta, new, old, usn):
 
     return False
 
-def update_secrets(newsecrets_ldb, secrets_ldb):
-    """Update secrets.ldb
-
-    :param newsecrets_ldb: An LDB object that is connected to the secrets.ldb
-                            of the reference provision
-    :param secrets_ldb: An LDB object that is connected to the secrets.ldb
-                            of the updated provision"""
-
-    message(SIMPLE, "update secrets.ldb")
-    reference = newsecrets_ldb.search(expression="dn=@MODULES", base="",
-                                        scope=SCOPE_SUBTREE)
-    current = secrets_ldb.search(expression="dn=@MODULES", base="",
-                                        scope=SCOPE_SUBTREE)
-    delta = secrets_ldb.msg_diff(current[0], reference[0])
-    delta.dn = current[0].dn
-    secrets_ldb.modify(delta)
-
-    reference = newsecrets_ldb.search(expression="objectClass=top", base="",
-                                        scope=SCOPE_SUBTREE, attrs=["dn"])
-    current = secrets_ldb.search(expression="objectClass=top", base="",
-                                        scope=SCOPE_SUBTREE, attrs=["dn"])
-    hash_new = {}
-    hash = {}
-    listMissing = []
-    listPresent = []
-
-    empty = Message()
-    for i in range(0, len(reference)):
-        hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]
-
-    # Create a hash for speeding the search of existing object in the
-    # current provision
-    for i in range(0, len(current)):
-        hash[str(current[i]["dn"]).lower()] = current[i]["dn"]
-
-    for k in hash_new.keys():
-        if not hash.has_key(k):
-            listMissing.append(hash_new[k])
-        else:
-            listPresent.append(hash_new[k])
-
-    for entry in listMissing:
-        reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        delta = secrets_ldb.msg_diff(empty,reference[0])
-        for att in hashAttrNotCopied.keys():
-            delta.remove(att)
-        message(CHANGE, "Entry %s is missing from secrets.ldb"%reference[0].dn)
-        for att in delta:
-            message(CHANGE, " Adding attribute %s"%att)
-        delta.dn = reference[0].dn
-        secrets_ldb.add(delta)
-
-    for entry in listPresent:
-        reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        delta = secrets_ldb.msg_diff(current[0],reference[0])
-        for att in hashAttrNotCopied.keys():
-            delta.remove(att)
-        for att in delta:
-            if att == "name":
-                message(CHANGE, "Found attribute name on  %s, must rename the DN "%(current[0].dn))
-                identic_rename(secrets_ldb,reference[0].dn)
-            else:
-                delta.remove(att)
-
-    for entry in listPresent:
-        reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
-        delta = secrets_ldb.msg_diff(current[0],reference[0])
-        for att in hashAttrNotCopied.keys():
-            delta.remove(att)
-        for att in delta:
-            if att != "dn":
-                message(CHANGE, " Adding/Changing attribute %s to %s"%(att,current[0].dn))
-
-        delta.dn = current[0].dn
-        secrets_ldb.modify(delta)
-
-
 def dump_denied_change(dn, att, flagtxt, current, reference):
-    """Print detailed information about why a changed is denied
+    """Print detailed information about why a change is denied
 
     :param dn: DN of the object which attribute is denied
     :param att: Attribute that was supposed to be upgraded
@@ -465,61 +449,100 @@ def dump_denied_change(dn, att, flagtxt, current, reference):
     :param reference: Value(s) of the reference attribute"""
 
     message(CHANGE, "dn= " + str(dn)+" " + att+" with flag " + flagtxt
-                +" is not allowed to be changed/removed, I discard this change")
-    if att != "objectSid" :
+                + " must not be changed/removed. Discarding the change")
+    if att == "objectSid" :
+        message(CHANGE, "old : %s" % ndr_unpack(security.dom_sid, current[0]))
+        message(CHANGE, "new : %s" % ndr_unpack(security.dom_sid, reference[0]))
+    elif att == "rIDPreviousAllocationPool" or att == "rIDAllocationPool":
+        message(CHANGE, "old : %s" % int64range2str(current[0]))
+        message(CHANGE, "new : %s" % int64range2str(reference[0]))
+    else:
         i = 0
         for e in range(0, len(current)):
             message(CHANGE, "old %d : %s" % (i, str(current[e])))
             i+=1
-        if reference != None:
+        if reference is not None:
             i = 0
             for e in range(0, len(reference)):
                 message(CHANGE, "new %d : %s" % (i, str(reference[e])))
                 i+=1
-    else:
-        message(CHANGE, "old : %s" % str(ndr_unpack( security.dom_sid, current[0])))
-        message(CHANGE, "new : %s" % str(ndr_unpack( security.dom_sid, reference[0])))
-
 
 def handle_special_add(samdb, dn, names):
     """Handle special operation (like remove) on some object needed during
-       upgrade
+    upgrade
 
     This is mostly due to wrong creation of the object in previous provision.
     :param samdb: An Ldb object representing the SAM database
     :param dn: DN of the object to inspect
-    :param names: list of key provision parameters"""
+    :param names: list of key provision parameters
+    """
 
     dntoremove = None
-    if str(dn).lower() == ("CN=IIS_IUSRS, CN=Builtin, %s" % names.rootdn).lower():
+    objDn = Dn(samdb, "CN=IIS_IUSRS, CN=Builtin, %s" % names.rootdn)
+    if dn == objDn :
         #This entry was misplaced lets remove it if it exists
         dntoremove = "CN=IIS_IUSRS, CN=Users, %s" % names.rootdn
 
-    objname = "CN=Certificate Service DCOM Access, CN=Builtin, %s" % names.rootdn
-    if str(dn).lower() == objname.lower():
+    objDn = Dn(samdb,
+                "CN=Certificate Service DCOM Access, CN=Builtin, %s" % names.rootdn)
+    if dn == objDn:
         #This entry was misplaced lets remove it if it exists
         dntoremove = "CN=Certificate Service DCOM Access,"\
                      "CN=Users, %s" % names.rootdn
 
-    objname = "CN=Cryptographic Operators, CN=Builtin, %s" % names.rootdn
-    if str(dn).lower() == objname.lower():
+    objDn = Dn(samdb, "CN=Cryptographic Operators, CN=Builtin, %s" % names.rootdn)
+    if dn == objDn:
         #This entry was misplaced lets remove it if it exists
         dntoremove = "CN=Cryptographic Operators, CN=Users, %s" % names.rootdn
 
-    objname = "CN=Event Log Readers, CN=Builtin, %s" % names.rootdn
-    if str(dn).lower() == objname.lower():
+    objDn = Dn(samdb, "CN=Event Log Readers, CN=Builtin, %s" % names.rootdn)
+    if dn == objDn:
         #This entry was misplaced lets remove it if it exists
         dntoremove = "CN=Event Log Readers, CN=Users, %s" % names.rootdn
 
-    if dntoremove != None:
+    objDn = Dn(samdb,"CN=System,CN=WellKnown Security Principals,"
+                     "CN=Configuration,%s" % names.rootdn)
+    if dn == objDn:
+        oldDn = Dn(samdb,"CN=Well-Known-Security-Id-System,"
+                         "CN=WellKnown Security Principals,"
+                         "CN=Configuration,%s" % names.rootdn)
+
+        res = samdb.search(expression="(dn=%s)" % oldDn,
+                            base=str(names.rootdn),
+                            scope=SCOPE_SUBTREE, attrs=["dn"],
+                            controls=["search_options:1:2"])
+
+        res2 = samdb.search(expression="(dn=%s)" % dn,
+                            base=str(names.rootdn),
+                            scope=SCOPE_SUBTREE, attrs=["dn"],
+                            controls=["search_options:1:2"])
+
+        if len(res) > 0 and len(res2) == 0:
+            message(CHANGE, "Existing object %s must be replaced by %s. "
+                            "Renaming old object" % (str(oldDn), str(dn)))
+            samdb.rename(oldDn, objDn, ["relax:0", "provision:0"])
+
+        return 0
+
+    if dntoremove is not None:
+        res = samdb.search(expression="(cn=RID Set)",
+                            base=str(names.rootdn),
+                            scope=SCOPE_SUBTREE, attrs=["dn"],
+                            controls=["search_options:1:2"])
+
+        if len(res) == 0:
+            return 2
         res = samdb.search(expression="(dn=%s)" % dntoremove,
                             base=str(names.rootdn),
                             scope=SCOPE_SUBTREE, attrs=["dn"],
                             controls=["search_options:1:2"])
         if len(res) > 0:
-            message(CHANGE, "Existing object %s must be replaced by %s,"\
-                            "removing old object" % (dntoremove, str(dn)))
+            message(CHANGE, "Existing object %s must be replaced by %s. "
+                            "Removing old object" % (dntoremove, str(dn)))
             samdb.delete(res[0]["dn"])
+            return 0
+
+    return 1
 
 
 def check_dn_nottobecreated(hash, index, listdn):
@@ -538,7 +561,7 @@ def check_dn_nottobecreated(hash, index, listdn):
     :param listdn: List of DNs on which the current DN depends on
     :return: None if the current object do not depend on other
               object or if all object have been created before."""
-    if listdn == None:
+    if listdn is None:
         return None
     for dn in listdn:
         key = str(dn).lower()
@@ -566,30 +589,54 @@ def add_missing_object(ref_samdb, samdb, dn, names, basedn, hash, index):
     :param index: Current creation order
     :return: True if the object was created False otherwise"""
 
-    handle_special_add(samdb, dn, names)
+    ret = handle_special_add(samdb, dn, names)
+
+    if ret == 2:
+        return False
+
+    if ret == 0:
+        return True
+
+
     reference = ref_samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
                     scope=SCOPE_SUBTREE, controls=["search_options:1:2"])
     empty = Message()
     delta = samdb.msg_diff(empty, reference[0])
     delta.dn
-    for att in hashAttrNotCopied.keys():
-        delta.remove(att)
-    for att in backlinked:
-        delta.remove(att)
-    depend_on_yettobecreated = None
-    for att in dn_syntax_att:
-        depend_on_yet_tobecreated = check_dn_nottobecreated(hash, index,
-                                                            delta.get(str(att)))
-        if depend_on_yet_tobecreated != None:
-            message(CHANGE, "Object %s depends on %s in attribute %s," \
-                            "delaying the creation" % (str(dn), \
-                                      depend_on_yet_tobecreated, str(att)))
-            return False
+    skip = False
+    try:
+        if str(reference[0].get("cn"))  == "RID Set":
+            for klass in reference[0].get("objectClass"):
+                if str(klass).lower() == "ridset":
+                    skip = True
+    finally:
+        if delta.get("objectSid"):
+            sid = str(ndr_unpack(security.dom_sid, str(reference[0]["objectSid"])))
+            m = re.match(r".*-(\d+)$", sid)
+            if m and int(m.group(1))>999:
+                delta.remove("objectSid")
+        for att in attrNotCopied:
+            delta.remove(att)
+        for att in backlinked:
+            delta.remove(att)
+        depend_on_yettobecreated = None
+        for att in dn_syntax_att:
+            depend_on_yet_tobecreated = check_dn_nottobecreated(hash, index,
+                                                                delta.get(str(att)))
+            if depend_on_yet_tobecreated is not None:
+                message(CHANGE, "Object %s depends on %s in attribute %s. "
+                                "Delaying the creation" % (dn,
+                                          depend_on_yet_tobecreated, att))
+                return False
 
-    delta.dn = dn
-    message(CHANGE,"Object %s will be added" % dn)
-    samdb.add(delta, ["relax:0"])
-    return True
+        delta.dn = dn
+        if not skip:
+            message(CHANGE,"Object %s will be added" % dn)
+            samdb.add(delta, ["relax:0", "provision:0"])
+        else:
+            message(CHANGE,"Object %s was skipped" % dn)
+
+        return True
 
 def gen_dn_index_hash(listMissing):
     """Generate a hash associating the DN to its creation order
@@ -619,22 +666,27 @@ def add_deletedobj_containers(ref_samdb, samdb, names):
         ref_delObjCnt = ref_samdb.search(expression="(cn=Deleted Objects)",
                                             base=part, scope=SCOPE_SUBTREE,
                                             attrs=["dn"],
-                                            controls=["show_deleted:0"])
+                                            controls=["show_deleted:0",
+                                                      "show_recycled:0"])
         delObjCnt = samdb.search(expression="(cn=Deleted Objects)",
                                     base=part, scope=SCOPE_SUBTREE,
                                     attrs=["dn"],
-                                    controls=["show_deleted:0"])
+                                    controls=["show_deleted:0",
+                                              "show_recycled:0"])
         if len(ref_delObjCnt) > len(delObjCnt):
             reference = ref_samdb.search(expression="cn=Deleted Objects",
                                             base=part, scope=SCOPE_SUBTREE,
-                                            controls=["show_deleted:0"])
+                                            controls=["show_deleted:0",
+                                                      "show_recycled:0"])
             empty = Message()
             delta = samdb.msg_diff(empty, reference[0])
 
             delta.dn = Dn(samdb, str(reference[0]["dn"]))
-            for att in hashAttrNotCopied.keys():
+            for att in attrNotCopied:
                 delta.remove(att)
-            samdb.add(delta)
+
+            modcontrols = ["relax:0", "provision:0"]
+            samdb.add(delta, modcontrols)
 
             listwko = []
             res = samdb.search(expression="(objectClass=*)", base=part,
@@ -642,7 +694,7 @@ def add_deletedobj_containers(ref_samdb, samdb, names):
                                attrs=["dn", "wellKnownObjects"])
 
             targetWKO = "%s:%s" % (wkoPrefix, str(reference[0]["dn"]))
-            found = 0
+            found = False
 
             if len(res[0]) > 0:
                 wko = res[0]["wellKnownObjects"]
@@ -650,7 +702,7 @@ def add_deletedobj_containers(ref_samdb, samdb, names):
                 # The wellKnownObject that we want to add.
                 for o in wko:
                     if str(o) == targetWKO:
-                        found = 1
+                        found = True
                     listwko.append(str(o))
 
             if not found:
@@ -694,8 +746,9 @@ def add_missing_entries(ref_samdb, samdb, names, basedn, list):
                 # DN can't be created because it depends on some
                 # other DN in the list
                 listDefered.append(dn)
+
     if len(listDefered) != 0:
-        raise ProvisioningError("Unable to insert missing elements:" \
+        raise ProvisioningError("Unable to insert missing elements: "
                                 "circular references")
 
 def handle_links(samdb, att, basedn, dn, value, ref_value, delta):
@@ -710,16 +763,16 @@ def handle_links(samdb, att, basedn, dn, value, ref_value, delta):
     :param delta: The MessageElement object that will be applied for
                    transforming the current provision"""
 
-    res = samdb.search(expression="dn=%s" % dn, base=basedn,
-                        controls=["search_options:1:2", "reveal:1"],
+    res = samdb.search(base=dn, controls=["search_options:1:2", "reveal:1"],
                         attrs=[att])
 
     blacklist = {}
     hash = {}
     newlinklist = []
-    changed = 0
+    changed = False
 
-    newlinklist.extend(value)
+    for v in value:
+        newlinklist.append(str(v))
 
     for e in value:
         hash[e] = 1
@@ -738,13 +791,207 @@ def handle_links(samdb, att, basedn, dn, value, ref_value, delta):
     for e in ref_value:
         if not blacklist.has_key(e) and not hash.has_key(e):
             newlinklist.append(str(e))
-            changed = 1
+            changed = True
     if changed:
         delta[att] = MessageElement(newlinklist, FLAG_MOD_REPLACE, att)
     else:
         delta.remove(att)
 
-def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
+    return delta
+
+
+msg_elt_flag_strs = {
+    ldb.FLAG_MOD_ADD: "MOD_ADD",
+    ldb.FLAG_MOD_REPLACE: "MOD_REPLACE",
+    ldb.FLAG_MOD_DELETE: "MOD_DELETE" }
+
+def checkKeepAttributeOldMtd(delta, att, reference, current,
+                                    basedn, samdb):
+    """ Check if we should keep the attribute modification or not.
+        This function didn't use replicationMetadata to take a decision.
+
+        :param delta: A message diff object
+        :param att: An attribute
+        :param reference: A message object for the current entry comming from
+                            the reference provision.
+        :param current: A message object for the current entry commin from
+                            the current provision.
+        :param basedn: The DN of the partition
+        :param samdb: A ldb connection to the sam database of the current provision.
+
+        :return: The modified message diff.
+    """
+    # Old school way of handling things for pre alpha12 upgrade
+    global defSDmodified
+    isFirst = False
+    txt = ""
+    dn = current[0].dn
+
+    for att in list(delta):
+        msgElt = delta.get(att)
+
+        if att == "nTSecurityDescriptor":
+            defSDmodified = True
+            delta.remove(att)
+            continue
+
+        if att == "dn":
+            continue
+
+        if not hashOverwrittenAtt.has_key(att):
+            if msgElt.flags() != FLAG_MOD_ADD:
+                if not handle_special_case(att, delta, reference, current,
+                                            False, basedn, samdb):
+                    if opts.debugchange or opts.debugall:
+                        try:
+                            dump_denied_change(dn, att,
+                                msg_elt_flag_strs[msgElt.flags()],
+                                current[0][att], reference[0][att])
+                        except KeyError:
+                            dump_denied_change(dn, att,
+                                msg_elt_flag_strs[msgElt.flags()],
+                                current[0][att], None)
+                    delta.remove(att)
+                continue
+        else:
+            if hashOverwrittenAtt.get(att)&2**msgElt.flags() :
+                continue
+            elif  hashOverwrittenAtt.get(att)==never:
+                delta.remove(att)
+                continue
+
+    return delta
+
+def checkKeepAttributeWithMetadata(delta, att, message, reference, current,
+                                    hash_attr_usn, basedn, usns, samdb):
+    """ Check if we should keep the attribute modification or not
+
+        :param delta: A message diff object
+        :param att: An attribute
+        :param message: A function to print messages
+        :param reference: A message object for the current entry comming from
+                            the reference provision.
+        :param current: A message object for the current entry commin from
+                            the current provision.
+        :param hash_attr_usn: A dictionnary with attribute name as keys,
+                                USN and invocation id as values.
+        :param basedn: The DN of the partition
+        :param usns: A dictionnary with invocation ID as keys and USN ranges
+                     as values.
+        :param samdb: A ldb object pointing to the sam DB
+
+        :return: The modified message diff.
+    """
+    global defSDmodified
+    isFirst = True
+    txt = ""
+    dn = current[0].dn
+
+    for att in list(delta):
+        if att in ["dn", "objectSid"]:
+            delta.remove(att)
+            continue
+
+        # We have updated by provision usn information so let's exploit
+        # replMetadataProperties
+        if att in forwardlinked:
+            curval = current[0].get(att, ())
+            refval = reference[0].get(att, ())
+            delta = handle_links(samdb, att, basedn, current[0]["dn"],
+                            curval, refval, delta)
+            continue
+
+
+        if isFirst and len(list(delta)) > 1:
+            isFirst = False
+            txt = "%s\n" % (str(dn))
+
+        if handle_special_case(att, delta, reference, current, True, None, None):
+            # This attribute is "complicated" to handle and handling
+            # was done in handle_special_case
+            continue
+
+        attrUSN = None
+        if hash_attr_usn.get(att):
+            [attrUSN, attInvId] = hash_attr_usn.get(att)
+
+        if attrUSN is None:
+            # If it's a replicated attribute and we don't have any USN
+            # information about it. It means that we never saw it before
+            # so let's add it !
+            # If it is a replicated attribute but we are not master on it
+            # (ie. not initially added in the provision we masterize).
+            # attrUSN will be -1
+            if isReplicated(att):
+                continue
+            else:
+                message(CHANGE, "Non replicated attribute %s changed" % att)
+                continue
+
+        if att == "nTSecurityDescriptor":
+            cursd = ndr_unpack(security.descriptor,
+                str(current[0]["nTSecurityDescriptor"]))
+            cursddl = cursd.as_sddl(names.domainsid)
+            refsd = ndr_unpack(security.descriptor,
+                str(reference[0]["nTSecurityDescriptor"]))
+            refsddl = refsd.as_sddl(names.domainsid)
+
+            diff = get_diff_sddls(refsddl, cursddl)
+            if diff == "":
+                # FIXME find a way to have it only with huge huge verbose mode
+                # message(CHANGE, "%ssd are identical" % txt)
+                # txt = ""
+                delta.remove(att)
+                continue
+            else:
+                delta.remove(att)
+                message(CHANGESD, "%ssd are not identical:\n%s" % (txt, diff))
+                txt = ""
+                if attrUSN == -1:
+                    message(CHANGESD, "But the SD has been changed by someonelse "\
+                                    "so it's impossible to know if the difference"\
+                                    " cames from the modification or from a previous bug")
+                    dnNotToRecalculate.append(str(dn))
+                else:
+                    dnToRecalculate.append(str(dn))
+                continue
+
+        if attrUSN == -1:
+            # This attribute was last modified by another DC forget
+            # about it
+            message(CHANGE, "%sAttribute: %s has been "
+                    "created/modified/deleted by another DC. "
+                    "Doing nothing" % (txt, att))
+            txt = ""
+            delta.remove(att)
+            continue
+        elif not usn_in_range(int(attrUSN), usns.get(attInvId)):
+            message(CHANGE, "%sAttribute: %s was not "
+                            "created/modified/deleted during a "
+                            "provision or upgradeprovision. Current "
+                            "usn: %d. Doing nothing" % (txt, att,
+                                                        attrUSN))
+            txt = ""
+            delta.remove(att)
+            continue
+        else:
+            if att == "defaultSecurityDescriptor":
+                defSDmodified = True
+            if attrUSN:
+                message(CHANGE, "%sAttribute: %s will be modified"
+                                "/deleted it was last modified "
+                                "during a provision. Current usn: "
+                                "%d" % (txt, att, attrUSN))
+                txt = ""
+            else:
+                message(CHANGE, "%sAttribute: %s will be added because "
+                                "it did not exist before" % (txt, att))
+                txt = ""
+            continue
+
+    return delta
+
+def update_present(ref_samdb, samdb, basedn, listPresent, usns):
     """ This function updates the object that are already present in the
         provision
 
@@ -754,10 +1001,9 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
                    (ie. DC=foo, DC=bar)
     :param listPresent: A list of object that is present in the provision
     :param usns: A list of USN range modified by previous provision and
-                 upgradeprovision
-    :param invocationid: The value of the invocationid for the current DC"""
+                 upgradeprovision grouped by invocation ID
+    """
 
-    global defSDmodified
     # This hash is meant to speedup lookup of attribute name from an oid,
     # it's for the replPropertyMetaData handling
     hash_oid_name = {}
@@ -773,7 +1019,9 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
         raise ProvisioningError(msg)
 
     changed = 0
-    controls = ["search_options:1:2", "sd_flags:1:2"]
+    controls = ["search_options:1:2", "sd_flags:1:0"]
+    if usns is not None:
+            message(CHANGE, "Using replPropertyMetadata for change selection")
     for dn in listPresent:
         reference = ref_samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
                                         scope=SCOPE_SUBTREE,
@@ -785,25 +1033,30 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
              (str(current[0].dn) != str(reference[0].dn)) and
              (str(current[0].dn).upper() == str(reference[0].dn).upper())
            ):
-            message(CHANGE, "Name are the same but case change,"\
-                            "let's rename %s to %s" % (str(current[0].dn),
-                                                       str(reference[0].dn)))
+            message(CHANGE, "Names are the same except for the case. "
+                            "Renaming %s to %s" % (str(current[0].dn),
+                                                   str(reference[0].dn)))
             identic_rename(samdb, reference[0].dn)
             current = samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
                                     scope=SCOPE_SUBTREE,
-                                    controls=["search_options:1:2"])
+                                    controls=controls)
 
         delta = samdb.msg_diff(current[0], reference[0])
 
-        for att in hashAttrNotCopied.keys():
+        for att in backlinked:
             delta.remove(att)
 
-        for att in backlinked:
+        for att in attrNotCopied:
             delta.remove(att)
 
         delta.remove("name")
 
-        if len(delta.items()) > 1 and usns != None:
+        nb_items = len(list(delta))
+
+        if nb_items == 1:
+            continue
+
+        if nb_items > 1 and usns is not None:
             # Fetch the replPropertyMetaData
             res = samdb.search(expression="dn=%s" % (str(dn)), base=basedn,
                                 scope=SCOPE_SUBTREE, controls=controls,
@@ -816,126 +1069,64 @@ def update_present(ref_samdb, samdb, basedn, listPresent, usns, invocationid):
                 # We put in this hash only modification
                 # made on the current host
                 att = hash_oid_name[samdb.get_oid_from_attid(o.attid)]
-                if str(o.originating_invocation_id) == str(invocationid):
-                    hash_attr_usn[att] = o.originating_usn
+                if str(o.originating_invocation_id) in usns.keys():
+                    hash_attr_usn[att] = [o.originating_usn, str(o.originating_invocation_id)]
                 else:
-                    hash_attr_usn[att] = -1
-
-        isFirst = 0
-        txt = ""
-        for att in delta:
-            if usns != None:
-                if forwardlinked.has_key(att):
-                    handle_links(samdb, att, basedn, current[0]["dn"],
-                                    current[0][att], reference[0][att], delta)
-
-                if isFirst == 0 and len(delta.items())>1:
-                    isFirst = 1
-                    txt = "%s\n" % (str(dn))
-                if att == "dn":
-                    continue
-                if att == "rIDAvailablePool":
-                    delta.remove(att)
-                    continue
-                if att == "objectSid":
-                    delta.remove(att)
-                    continue
-                if att == "creationTime":
-                    delta.remove(att)
-                    continue
-                if att == "oEMInformation":
-                    delta.remove(att)
-                    continue
-                if att == "msDs-KeyVersionNumber":
-                    delta.remove(att)
-                    continue
-                if handle_special_case(att, delta, reference, current, usns):
-                    # This attribute is "complicated" to handle and handling
-                    # was done in handle_special_case
-                    continue
-                attrUSN = hash_attr_usn.get(att)
-                if att == "forceLogoff" and attrUSN == None:
-                    continue
-                if  attrUSN == None:
-                    delta.remove(att)
-                    continue
+                    hash_attr_usn[att] = [-1, None]
 
-                if attrUSN == -1:
-                    # This attribute was last modified by another DC forget
-                    # about it
-                    message(CHANGE, "%sAttribute: %s has been" \
-                            "created/modified/deleted  by another DC,"
-                            " do nothing" % (txt, att ))
-                    txt = ""
-                    delta.remove(att)
-                    continue
-                elif usn_in_range(int(attrUSN), usns) == 0:
-                    message(CHANGE, "%sAttribute: %s has been" \
-                                    "created/modified/deleted not during a" \
-                                    " provision or upgradeprovision: current" \
-                                    " usn %d , do nothing" % (txt, att, attrUSN))
-                    txt = ""
-                    delta.remove(att)
-                    continue
-                else:
-                    if att == "defaultSecurityDescriptor":
-                        defSDmodified = 1
-                    if attrUSN:
-                        message(CHANGE, "%sAttribute: %s will be modified" \
-                                        "/deleted it was last modified" \
-                                        "during a provision, current usn:" \
-                                        "%d" % (txt, att,  attrUSN))
-                        txt = ""
-                    else:
-                        message(CHANGE, "%sAttribute: %s will be added because" \
-                                        " it hasn't existed before " % (txt, att))
-                        txt = ""
-                    continue
+        if usns is not None:
+            delta = checkKeepAttributeWithMetadata(delta, att, message, reference,
+                                                    current, hash_attr_usn,
+                                                    basedn, usns, samdb)
+        else:
+            delta =  checkKeepAttributeOldMtd(delta, att, reference, current, basedn, samdb)
 
-            else:
-            # Old school way of handling things for pre alpha12 upgrade
-                defSDmodified = 1
-                msgElt = delta.get(att)
+        delta.dn = dn
 
-                if att == "nTSecurityDescriptor":
-                    delta.remove(att)
-                    continue
-
-                if att == "dn":
-                    continue
-
-                if not hashOverwrittenAtt.has_key(att):
-                    if msgElt.flags() != FLAG_MOD_ADD:
-                        if not handle_special_case(att, delta, reference, current,
-                                                    usns):
-                            if opts.debugchange or opts.debugall:
-                                try:
-                                    dump_denied_change(dn, att,
-                                        messageEltFlagToString(msgElt.flags()),
-                                        current[0][att], reference[0][att])
-                                except KeyError:
-                                    dump_denied_change(dn, att,
-                                        messageEltFlagToString(msgElt.flags()),
-                                        current[0][att], None)
-                            delta.remove(att)
-                        continue
-                else:
-                    if hashOverwrittenAtt.get(att)&2**msgElt.flags() :
-                        continue
-                    elif  hashOverwrittenAtt.get(att)==never:
-                        delta.remove(att)
-                        continue
 
-        delta.dn = dn
-        if len(delta.items()) >1:
-            attributes=", ".join(delta.keys())
-            message(CHANGE, "%s is different from the reference one, changed" \
+        if len(delta) >1:
+            # Skip dn as the value is not really changed ...
+            attributes=", ".join(delta.keys()[1:])
+            modcontrols = []
+            relaxedatt = ['iscriticalsystemobject', 'grouptype']
+            # Let's try to reduce as much as possible the use of relax control
+            for attr in delta.keys():
+                if attr.lower() in relaxedatt:
+                    modcontrols = ["relax:0", "provision:0"]
+            message(CHANGE, "%s is different from the reference one, changed"
                             " attributes: %s\n" % (dn, attributes))
-            changed = changed + 1
-            samdb.modify(delta)
+            changed += 1
+            samdb.modify(delta, modcontrols)
     return changed
 
-def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs):
+def reload_full_schema(samdb, names):
+    """Load the updated schema with all the new and existing classes
+       and attributes.
+
+    :param samdb: An LDB object connected to the sam.ldb of the update
+                  provision
+    :param names: List of key provision parameters
+    """
+
+    schemadn = str(names.schemadn)
+    current = samdb.search(expression="objectClass=*", base=schemadn,
+                                scope=SCOPE_SUBTREE)
+    schema_ldif = ""
+    prefixmap_data = ""
+
+    for ent in current:
+        schema_ldif += samdb.write_ldif(ent, ldb.CHANGETYPE_NONE)
+
+    prefixmap_data = open(setup_path("prefixMap.txt"), 'r').read()
+    prefixmap_data = b64encode(prefixmap_data)
+
+    # We don't actually add this ldif, just parse it
+    prefixmap_ldif = "dn: %s\nprefixMap:: %s\n\n" % (schemadn, prefixmap_data)
+
+    dsdb._dsdb_set_schema_from_ldif(samdb, prefixmap_ldif, schema_ldif, schemadn)
+
+
+def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs, prereloadfunc):
     """Check differences between the reference provision and the upgraded one.
 
     It looks for all objects which base DN is name.
@@ -950,8 +1141,11 @@ def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs):
     :param basedn: String value of the DN of the partition
     :param names: List of key provision parameters
     :param schema: A Schema object
-    :param provisionUSNs:  The USNs modified by provision/upgradeprovision
-                           last time"""
+    :param provisionUSNs:  A dictionnary with range of USN modified during provision
+                            or upgradeprovision. Ranges are grouped by invocationID.
+    :param prereloadfunc: A function that must be executed just before the reload
+                  of the schema
+    """
 
     hash_new = {}
     hash = {}
@@ -996,14 +1190,21 @@ def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs):
     # a complete schema is needed as the insertion of attributes
     # and class is done against it
     # and the schema is self validated
-    samdb.set_schema_from_ldb(schema.ldb)
+    samdb.set_schema(schema)
     try:
         message(SIMPLE, "There are %d missing objects" % (len(listMissing)))
         add_deletedobj_containers(ref_samdb, samdb, names)
 
         add_missing_entries(ref_samdb, samdb, names, basedn, listMissing)
+
+        prereloadfunc()
+        message(SIMPLE, "Reloading a merged schema, which might trigger "
+                        "reindexing so please be patient")
+        reload_full_schema(samdb, names)
+        message(SIMPLE, "Schema reloaded!")
+
         changed = update_present(ref_samdb, samdb, basedn, listPresent,
-                                    provisionUSNs, names.invocation)
+                                    provisionUSNs)
         message(SIMPLE, "There are %d changed objects" % (changed))
         return 1
 
@@ -1014,7 +1215,6 @@ def update_partition(ref_samdb, samdb, basedn, names, schema, provisionUSNs):
         return 0
 
 
-
 def check_updated_sd(ref_sam, cur_sam, names):
     """Check if the security descriptor in the upgraded provision are the same
        as the reference
@@ -1038,6 +1238,7 @@ def check_updated_sd(ref_sam, cur_sam, names):
                     str(reference[i]["nTSecurityDescriptor"]))
         hash[str(reference[i]["dn"]).lower()] = refsd.as_sddl(names.domainsid)
 
+
     for i in range(0, len(current)):
         key = str(current[i]["dn"]).lower()
         if hash.has_key(key):
@@ -1045,9 +1246,9 @@ def check_updated_sd(ref_sam, cur_sam, names):
                         str(current[i]["nTSecurityDescriptor"]))
             sddl = cursd.as_sddl(names.domainsid)
             if sddl != hash[key]:
-                txt = get_diff_sddls(hash[key], sddl)
+                txt = get_diff_sddls(hash[key], sddl, False)
                 if txt != "":
-                    message(CHANGESD, "On object %s ACL is different"\
+                    message(CHANGESD, "On object %s ACL is different"
                                       " \n%s" % (current[i]["dn"], txt))
 
 
@@ -1059,37 +1260,38 @@ def fix_partition_sd(samdb, names):
     :param samdb: An LDB object pointing to the sam of the current provision
     :param names: A list of key provision parameters
     """
+    alwaysRecalculate = False
+    if len(dnToRecalculate) == 0 and len(dnNotToRecalculate) == 0:
+        alwaysRecalculate = True
+
+
+    # NC's DN can't be both in dnToRecalculate and dnNotToRecalculate
     # First update the SD for the rootdn
-    res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
-                         scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                         controls=["search_options:1:2"])
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_domain_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor")
-    samdb.modify(delta, ["recalculate_sd:0"])
+    if alwaysRecalculate or str(names.rootdn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.rootdn))
+        descr = get_domain_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor")
+        samdb.modify(delta)
+
     # Then the config dn
-    res = samdb.search(expression="objectClass=*", base=str(names.configdn),
-                        scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                        controls=["search_options:1:2"])
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_config_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor" )
-    samdb.modify(delta, ["recalculate_sd:0"])
-    # Then the schema dn
-    res = samdb.search(expression="objectClass=*", base=str(names.schemadn),
-                        scope=SCOPE_BASE, attrs=["dn", "whenCreated"],
-                        controls=["search_options:1:2"])
+    if alwaysRecalculate or str(names.configdn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.configdn))
+        descr = get_config_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor" )
+        samdb.modify(delta)
 
-    delta = Message()
-    delta.dn = Dn(samdb, str(res[0]["dn"]))
-    descr = get_schema_descriptor(names.domainsid)
-    delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
-                                                    "nTSecurityDescriptor" )
-    samdb.modify(delta, ["recalculate_sd:0"])
+    # Then the schema dn
+    if alwaysRecalculate or str(names.schemadn) in dnToRecalculate:
+        delta = Message()
+        delta.dn = Dn(samdb, str(names.schemadn))
+        descr = get_schema_descriptor(names.domainsid)
+        delta["nTSecurityDescriptor"] = MessageElement(descr, FLAG_MOD_REPLACE,
+                                                        "nTSecurityDescriptor" )
+        samdb.modify(delta)
 
 def rebuild_sd(samdb, names):
     """Rebuild security descriptor of the current provision from scratch
@@ -1102,30 +1304,46 @@ def rebuild_sd(samdb, names):
 
     :param names: List of key provision parameters"""
 
+    fix_partition_sd(samdb, names)
 
+    # List of namming contexts
+    listNC = [str(names.rootdn), str(names.configdn), str(names.schemadn)]
     hash = {}
-    res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
+    if len(dnToRecalculate) == 0:
+        res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
                         scope=SCOPE_SUBTREE, attrs=["dn", "whenCreated"],
                         controls=["search_options:1:2"])
-    for obj in res:
-        if not (str(obj["dn"]) == str(names.rootdn) or
-            str(obj["dn"]) == str(names.configdn) or \
-            str(obj["dn"]) == str(names.schemadn)):
-            hash[str(obj["dn"])] = obj["whenCreated"]
-
-    listkeys = hash.keys()
-    listkeys.sort(dn_sort)
-
-    for key in listkeys:
+        for obj in res:
+                hash[str(obj["dn"])] = obj["whenCreated"]
+    else:
+        for dn in dnToRecalculate:
+            if hash.has_key(dn):
+                continue
+            # fetch each dn to recalculate and their child within the same partition
+            res = samdb.search(expression="objectClass=*", base=dn,
+                        scope=SCOPE_SUBTREE, attrs=["dn", "whenCreated"])
+            for obj in res:
+                hash[str(obj["dn"])] = obj["whenCreated"]
+
+    listKeys = list(set(hash.keys()))
+    listKeys.sort(dn_sort)
+
+    if len(dnToRecalculate) != 0:
+        message(CHANGESD, "%d DNs have been marked as needed to be recalculated"\
+                            ", recalculating %d due to inheritance"
+                            % (len(dnToRecalculate), len(listKeys)))
+
+    for key in listKeys:
+        if (key in listNC or
+                    key in dnNotToRecalculate):
+            continue
+        delta = Message()
+        delta.dn = Dn(samdb, key)
         try:
-            delta = Message()
-            delta.dn = Dn(samdb, key)
             delta["whenCreated"] = MessageElement(hash[key], FLAG_MOD_REPLACE,
                                                     "whenCreated" )
-            samdb.modify(delta, ["recalculate_sd:0"])
-        except:
-            # XXX: We should always catch an explicit exception.
-            # What could go wrong here?
+            samdb.modify(delta, ["recalculate_sd:0","relax:0"])
+        except LdbError, e:
             samdb.transaction_cancel()
             res = samdb.search(expression="objectClass=*", base=str(names.rootdn),
                                 scope=SCOPE_SUBTREE,
@@ -1133,14 +1351,23 @@ def rebuild_sd(samdb, names):
                                 controls=["search_options:1:2"])
             badsd = ndr_unpack(security.descriptor,
                         str(res[0]["nTSecurityDescriptor"]))
-            print "bad stuff %s"%badsd.as_sddl(names.domainsid)
+            message(ERROR, "On %s bad stuff %s" % (str(delta.dn),badsd.as_sddl(names.domainsid)))
             return
 
+def hasATProvision(samdb):
+        entry = samdb.search(expression="dn=@PROVISION", base = "",
+                                scope=SCOPE_BASE,
+                                attrs=["dn"])
+
+        if entry != None and len(entry) == 1:
+            return True
+        else:
+            return False
+
 def removeProvisionUSN(samdb):
         attrs = [samba.provision.LAST_PROVISION_USN_ATTRIBUTE, "dn"]
         entry = samdb.search(expression="dn=@PROVISION", base = "",
-                                scope=SCOPE_SUBTREE,
-                                controls=["search_options:1:2"],
+                                scope=SCOPE_BASE,
                                 attrs=attrs)
         empty = Message()
         empty.dn = entry[0].dn
@@ -1149,48 +1376,17 @@ def removeProvisionUSN(samdb):
         delta.dn = entry[0].dn
         samdb.modify(delta)
 
-def delta_update_basesamdb(refpaths, paths, creds, session, lp):
-    """Update the provision container db: sam.ldb
-    This function is aimed for alpha9 and newer;
-
-    :param refpaths: An object holding the different importants paths for
-                     reference provision object
-    :param paths: An object holding the different importants paths for
-                  upgraded provision object
-    :param creds: Credential used for openning LDB files
-    :param session: Session to use for openning LDB files
-    :param lp: A loadparam object"""
-
-    message(SIMPLE,
-            "Update base samdb by searching difference with reference one")
-    refsam = Ldb(refpaths.samdb, session_info=session, credentials=creds,
-                    lp=lp, options=["modules:"])
-    sam = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp,
-                options=["modules:"])
-
-    empty = Message()
+def remove_stored_generated_attrs(paths, creds, session, lp):
+    """Remove previously stored constructed attributes
 
-    reference = refsam.search(expression="")
-
-    for refentry in reference:
-        entry = sam.search(expression="dn=%s" % refentry["dn"],
-                            scope=SCOPE_SUBTREE)
-        if not len(entry):
-            message(CHANGE, "Adding %s to sam db" % str(delta.dn))
-            delta = sam.msg_diff(empty, refentry)
-            if str(refentry.dn) == "@PROVISION" and\
-                delta.get(samba.provision.LAST_PROVISION_USN_ATTRIBUTE):
-                delta.remove(samba.provision.LAST_PROVISION_USN_ATTRIBUTE)
-            delta.dn = refentry.dn
-            sam.add(delta)
-        else:
-            delta = sam.msg_diff(entry[0], refentry)
-            if str(refentry.dn) == "@PROVISION" and\
-                delta.get(samba.provision.LAST_PROVISION_USN_ATTRIBUTE):
-                delta.remove(samba.provision.LAST_PROVISION_USN_ATTRIBUTE)
-            if len(delta.items()) > 1:
-                delta.dn = refentry.dn
-                sam.modify(delta)
+    :param paths: List of paths for different provision objects
+                        from the upgraded provision
+    :param creds: A credential object
+    :param session: A session object
+    :param lp: A line parser object
+    :return: An associative array whose key are the different constructed
+             attributes and the value the dn where this attributes were found.
+     """
 
 
 def simple_update_basesamdb(newpaths, paths, names):
@@ -1241,7 +1437,7 @@ def update_privilege(ref_private_path, cur_private_path):
                 os.path.join(cur_private_path, "privilege.ldb"))
 
 
-def update_samdb(ref_samdb, samdb, names, highestUSN, schema):
+def update_samdb(ref_samdb, samdb, names, provisionUSNs, schema, prereloadfunc):
     """Upgrade the SAM DB contents for all the provision partitions
 
     :param ref_sambdb: An LDB object conntected to the sam.ldb of the reference
@@ -1249,13 +1445,16 @@ def update_samdb(ref_samdb, samdb, names, highestUSN, schema):
     :param samdb: An LDB object connected to the sam.ldb of the update
                   provision
     :param names: List of key provision parameters
-    :param highestUSN:  The highest USN modified by provision/upgradeprovision
-                        last time
-    :param schema: A Schema object that represent the schema of the provision"""
+    :param provisionUSNs:  A dictionnary with range of USN modified during provision
+                            or upgradeprovision. Ranges are grouped by invocationID.
+    :param schema: A Schema object that represent the schema of the provision
+    :param prereloadfunc: A function that must be executed just before the reload
+                  of the schema
+    """
 
     message(SIMPLE, "Starting update of samdb")
     ret = update_partition(ref_samdb, samdb, str(names.rootdn), names,
-                            schema, highestUSN)
+                            schema, provisionUSNs, prereloadfunc)
     if ret:
         message(SIMPLE, "Update of samdb finished")
         return 1
@@ -1264,236 +1463,530 @@ def update_samdb(ref_samdb, samdb, names, highestUSN, schema):
         return 0
 
 
-def update_machine_account_password(samdb, secrets_ldb, names):
-    """Update (change) the password of the current DC both in the SAM db and in
-       secret one
-
-    :param samdb: An LDB object related to the sam.ldb file of a given provision
-    :param secrets_ldb: An LDB object related to the secrets.ldb file of a given
-                        provision
-    :param names: List of key provision parameters"""
-
-    message(SIMPLE, "Update machine account")
-    expression = "samAccountName=%s$" % names.netbiosname
-    secrets_msg = secrets_ldb.search(expression=expression,
-                                        attrs=["secureChannelType"])
-    if int(secrets_msg[0]["secureChannelType"][0]) == SEC_CHAN_BDC:
-        res = samdb.search(expression=expression, attrs=[])
-        assert(len(res) == 1)
-
-        msg = Message(res[0].dn)
-        machinepass = samba.generate_random_password(128, 255)
-        msg["userPassword"] = MessageElement(machinepass, FLAG_MOD_REPLACE,
-                                                "userPassword")
-        samdb.modify(msg)
-
-        res = samdb.search(expression=("samAccountName=%s$" % names.netbiosname),
-                     attrs=["msDs-keyVersionNumber"])
-        assert(len(res) == 1)
-        kvno = int(str(res[0]["msDs-keyVersionNumber"]))
-        secChanType = int(secrets_msg[0]["secureChannelType"][0])
-
-        secretsdb_self_join(secrets_ldb, domain=names.domain,
-                    realm=names.realm or sambaopts._lp.get('realm'),
-                    domainsid=names.domainsid,
-                    dnsdomain=names.dnsdomain,
-                    netbiosname=names.netbiosname,
-                    machinepass=machinepass,
-                    key_version_number=kvno,
-                    secure_channel_type=secChanType)
-    else:
-        raise ProvisioningError("Unable to find a Secure Channel" \
-                                "of type SEC_CHAN_BDC")
-
-
-def update_gpo(paths, creds, session, names):
-    """Create missing GPO file object if needed
-
-    Set ACL correctly also.
+def copyxattrs(dir, refdir):
+    """ Copy owner, groups, extended ACL and NT acls from
+    a reference dir to a destination dir
+
+    Both dir are supposed to hold the same files
+    :param dir: Destination dir
+    :param refdir: Reference directory"""
+
+    noxattr = 0
+    for root, dirs, files in os.walk(dir, topdown=True):
+        for name in files:
+            subdir=root[len(dir):]
+            ref = os.path.join("%s%s" % (refdir, subdir), name)
+            statsinfo = os.stat(ref)
+            tgt = os.path.join(root, name)
+            try:
+
+                os.chown(tgt, statsinfo.st_uid, statsinfo.st_gid)
+                # Get the xattr attributes if any
+                try:
+                    attribute = samba.xattr_native.wrap_getxattr(ref,
+                                                 xattr.XATTR_NTACL_NAME)
+                    samba.xattr_native.wrap_setxattr(tgt,
+                                                 xattr.XATTR_NTACL_NAME,
+                                                 attribute)
+                except Exception:
+                    noxattr = 1
+                attribute = samba.xattr_native.wrap_getxattr(ref,
+                                                 "system.posix_acl_access")
+                samba.xattr_native.wrap_setxattr(tgt,
+                                                 "system.posix_acl_access",
+                                                  attribute)
+            except Exception:
+                continue
+        for name in dirs:
+            subdir=root[len(dir):]
+            ref = os.path.join("%s%s" % (refdir, subdir), name)
+            statsinfo = os.stat(ref)
+            tgt = os.path.join(root, name)
+            try:
+                os.chown(os.path.join(root, name), statsinfo.st_uid,
+                          statsinfo.st_gid)
+                try:
+                    attribute = samba.xattr_native.wrap_getxattr(ref,
+                                                 xattr.XATTR_NTACL_NAME)
+                    samba.xattr_native.wrap_setxattr(tgt,
+                                                 xattr.XATTR_NTACL_NAME,
+                                                 attribute)
+                except Exception:
+                    noxattr = 1
+                attribute = samba.xattr_native.wrap_getxattr(ref,
+                                                 "system.posix_acl_access")
+                samba.xattr_native.wrap_setxattr(tgt,
+                                                 "system.posix_acl_access",
+                                                  attribute)
+
+            except Exception:
+                continue
+
+
+def backup_provision(paths, dir):
+    """This function backup the provision files so that a rollback
+    is possible
+
+    :param paths: Paths to different objects
+    :param dir: Directory where to store the backup
     """
-    dir = getpolicypath(paths.sysvol, names.dnsdomain, names.policyid)
-    if not os.path.isdir(dir):
-        create_gpo_struct(dir)
-
-    dir = getpolicypath(paths.sysvol, names.dnsdomain, names.policyid_dc)
-    if not os.path.isdir(dir):
-        create_gpo_struct(dir)
-    samdb = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp)
-    set_gpo_acl(paths.sysvol, names.dnsdomain, names.domainsid,
-        names.domaindn, samdb, lp)
-
-
-def getOEMInfo(samdb, rootdn):
-    """Return OEM Information on the top level
-    Samba4 use to store version info in this field
-
-    :param samdb: An LDB object connect to sam.ldb
-    :param rootdn: Root DN of the domain
-    :return: The content of the field oEMInformation (if any)"""
-    res = samdb.search(expression="(objectClass=*)", base=str(rootdn),
-                            scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
-    if len(res) > 0:
-        info = res[0]["oEMInformation"]
-        return info
-    else:
-        return ""
 
-def updateOEMInfo(samdb, names):
-    res = samdb.search(expression="(objectClass=*)", base=str(names.rootdn),
-                            scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
-    if len(res) > 0:
-        info = res[0]["oEMInformation"]
-        info = "%s, upgrade to %s" % (info, version)
-        delta = Message()
-        delta.dn = Dn(samdb, str(res[0]["dn"]))
-        delta["oEMInformation"] = MessageElement(info, FLAG_MOD_REPLACE,
-            "oEMInformation" )
-        samdb.modify(delta)
+    shutil.copytree(paths.sysvol, os.path.join(dir, "sysvol"))
+    copyxattrs(os.path.join(dir, "sysvol"), paths.sysvol)
+    shutil.copy2(paths.samdb, dir)
+    shutil.copy2(paths.secrets, dir)
+    shutil.copy2(paths.idmapdb, dir)
+    shutil.copy2(paths.privilege, dir)
+    if os.path.isfile(os.path.join(paths.private_dir,"eadb.tdb")):
+        shutil.copy2(os.path.join(paths.private_dir,"eadb.tdb"), dir)
+    shutil.copy2(paths.smbconf, dir)
+    shutil.copy2(os.path.join(paths.private_dir,"secrets.keytab"), dir)
 
+    samldbdir = os.path.join(paths.private_dir, "sam.ldb.d")
+    if not os.path.isdir(samldbdir):
+        samldbdir = paths.private_dir
+        schemaldb = os.path.join(paths.private_dir, "schema.ldb")
+        configldb = os.path.join(paths.private_dir, "configuration.ldb")
+        usersldb = os.path.join(paths.private_dir, "users.ldb")
+        shutil.copy2(schemaldb, dir)
+        shutil.copy2(usersldb, dir)
+        shutil.copy2(configldb, dir)
+    else:
+        shutil.copytree(samldbdir, os.path.join(dir, "sam.ldb.d"))
+
+
+
+
+def sync_calculated_attributes(samdb, names):
+   """Synchronize attributes used for constructed ones, with the
+      old constructed that were stored in the database.
+
+      This apply for instance to msds-keyversionnumber that was
+      stored and that is now constructed from replpropertymetadata.
+
+      :param samdb: An LDB object attached to the currently upgraded samdb
+      :param names: Various key parameter about current provision.
+   """
+   listAttrs = ["msDs-KeyVersionNumber"]
+   hash = search_constructed_attrs_stored(samdb, names.rootdn, listAttrs)
+   if hash.has_key("msDs-KeyVersionNumber"):
+       increment_calculated_keyversion_number(samdb, names.rootdn,
+                                            hash["msDs-KeyVersionNumber"])
+
+# Synopsis for updateprovision
+# 1) get path related to provision to be update (called current)
+# 2) open current provision ldbs
+# 3) fetch the key provision parameter (domain sid, domain guid, invocationid
+#    of the DC ....)
+# 4) research of lastProvisionUSN in order to get ranges of USN modified
+#    by either upgradeprovision or provision
+# 5) creation of a new provision the latest version of provision script
+#    (called reference)
+# 6) get reference provision paths
+# 7) open reference provision ldbs
+# 8) setup helpers data that will help the update process
+# 9) update the privilege ldb by copying the one of referecence provision to
+#    the current provision
+# 10)get the oemInfo field, this field contains information about the different
+#    provision that have been done
+# 11)Depending  on whether oemInfo has the string "alpha9" or alphaxx (x as an
+#    integer) or none of this the following things are done
+#    A) When alpha9 or alphaxx is present
+#       The base sam.ldb file is updated by looking at the difference between
+#       referrence one and the current one. Everything is copied with the
+#       exception of lastProvisionUSN attributes.
+#    B) Other case (it reflect that that provision was done before alpha9)
+#       The base sam.ldb of the reference provision is copied over
+#       the current one, if necessary ldb related to partitions are moved
+#       and renamed
+# The highest used USN is fetched so that changed by upgradeprovision
+# usn can be tracked
+# 12)A Schema object is created, it will be used to provide a complete
+#    schema to current provision during update (as the schema of the
+#    current provision might not be complete and so won't allow some
+#    object to be created)
+# 13)Proceed to full update of sam DB (see the separate paragraph about i)
+# 14)The secrets db is updated by pull all the difference from the reference
+#    provision into the current provision
+# 15)As the previous step has most probably modified the password stored in
+#    in secret for the current DC, a new password is generated,
+#    the kvno is bumped and the entry in samdb is also updated
+# 16)For current provision older than alpha9, we must fix the SD a little bit
+#    administrator to update them because SD used to be generated with the
+#    system account before alpha9.
+# 17)The highest usn modified so far is searched in the database it will be
+#    the upper limit for usn modified during provision.
+#    This is done before potential SD recalculation because we do not want
+#    SD modified during recalculation to be marked as modified during provision
+#    (and so possibly remplaced at next upgradeprovision)
+# 18)Rebuilt SD if the flag indicate to do so
+# 19)Check difference between SD of reference provision and those of the
+#    current provision. The check is done by getting the sddl representation
+#    of the SD. Each sddl in chuncked into parts (user,group,dacl,sacl)
+#    Each part is verified separetly, for dacl and sacl ACL is splited into
+#    ACEs and each ACE is verified separately (so that a permutation in ACE
+#    didn't raise as an error).
+# 20)The oemInfo field is updated to add information about the fact that the
+#    provision has been updated by the upgradeprovision version xxx
+#    (the version is the one obtained when starting samba with the --version
+#    parameter)
+# 21)Check if the current provision has all the settings needed for dynamic
+#    DNS update to work (that is to say the provision is newer than
+#    january 2010). If not dns configuration file from reference provision
+#    are copied in a sub folder and the administrator is invited to
+#    do what is needed.
+# 22)If the lastProvisionUSN attribute was present it is updated to add
+#    the range of usns modified by the current upgradeprovision
+
+
+# About updating the sam DB
+# The update takes place in update_partition function
+# This function read both current and reference provision and list all
+# the available DN of objects
+# If the string representation of a DN in reference provision is
+# equal to the string representation of a DN in current provision
+# (without taking care of case) then the object is flaged as being
+# present. If the object is not present in current provision the object
+# is being flaged as missing in current provision. Object present in current
+# provision but not in reference provision are ignored.
+# Once the list of objects present and missing is done, the deleted object
+# containers are created in the differents partitions (if missing)
+#
+# Then the function add_missing_entries is called
+# This function will go through the list of missing entries by calling
+# add_missing_object for the given object. If this function returns 0
+# it means that the object needs some other object in order to be created
+# The object is reappended at the end of the list to be created later
+# (and preferably after all the needed object have been created)
+# The function keeps on looping on the list of object to be created until
+# it's empty or that the number of defered creation is equal to the number
+# of object that still needs to be created.
+
+# The function add_missing_object will first check if the object can be created.
+# That is to say that it didn't depends other not yet created objects
+# If requisit can't be fullfilled it exists with 0
+# Then it will try to create the missing entry by creating doing
+# an ldb_message_diff between the object in the reference provision and
+# an empty object.
+# This resulting object is filtered to remove all the back link attribute
+# (ie. memberOf) as they will be created by the other linked object (ie.
+# the one with the member attribute)
+# All attributes specified in the attrNotCopied array are
+# also removed it's most of the time generated attributes
+
+# After missing entries have been added the update_partition function will
+# take care of object that exist but that need some update.
+# In order to do so the function update_present is called with the list
+# of object that are present in both provision and that might need an update.
+
+# This function handle first case mismatch so that the DN in the current
+# provision have the same case as in reference provision
+
+# It will then construct an associative array consiting of attributes as
+# key and invocationid as value( if the originating invocation id is
+# different from the invocation id of the current DC the value is -1 instead).
+
+# If the range of provision modified attributes is present, the function will
+# use the replMetadataProperty update method which is the following:
+#  Removing attributes that should not be updated: rIDAvailablePool, objectSid,
+#   creationTime, msDs-KeyVersionNumber, oEMInformation
+#  Check for each attribute if its usn is within one of the modified by
+#   provision range and if its originating id is the invocation id of the
+#   current DC, then validate the update from reference to current.
+#   If not or if there is no replMetatdataProperty for this attribute then we
+#   do not update it.
+# Otherwise (case the range of provision modified attribute is not present) it
+# use the following process:
+#  All attributes that need to be added are accepted at the exeption of those
+#   listed in hashOverwrittenAtt, in this case the attribute needs to have the
+#   correct flags specified.
+#  For attributes that need to be modified or removed, a check is performed
+#  in OverwrittenAtt, if the attribute is present and the modification flag
+#  (remove, delete) is one of those listed for this attribute then modification
+#  is accepted. For complicated handling of attribute update, the control is passed
+#  to handle_special_case
 
-def setup_path(file):
-    return os.path.join(setup_dir, file)
 
 
 if __name__ == '__main__':
     global defSDmodified
-    defSDmodified = 0
+    defSDmodified = False
     # From here start the big steps of the program
-    # First get files paths
+    # 1) First get files paths
     paths = get_paths(param, smbconf=smbconf)
-    paths.setup = setup_dir
     # Get ldbs with the system session, it is needed for searching
     # provision parameters
     session = system_session()
 
     # This variable will hold the last provision USN once if it exists.
     minUSN = 0
-
+    # 2)
     ldbs = get_ldbs(paths, creds, session, lp)
-    ldbs.startTransactions()
-
-    # Guess all the needed names (variables in fact) from the current
-    # provision.
-    names = find_provision_key_parameters(ldbs.sam, ldbs.secrets, paths,
-                                            smbconf, lp)
-    lastProvisionUSNs = getLastProvisionUSN(ldbs.sam)
-    if lastProvisionUSNs != None:
-        message(CHANGE,
-            "Find a last provision USN, %d range(s)" % len(lastProvisionUSNs))
-
-    # Objects will be created with the admin session
-    # (not anymore system session)
-    adm_session = admin_session(lp, str(names.domainsid))
-    # So we reget handle on objects
-    # ldbs = get_ldbs(paths, creds, adm_session, lp)
-
-    if not sanitychecks(ldbs.sam, names):
-        message(SIMPLE, "Sanity checks for the upgrade fails, checks messages" \
-                        " and correct them before rerunning upgradeprovision")
-        sys.exit(1)
-
-    # Let's see provision parameters
-    print_provision_key_parameters(names)
+    backupdir = tempfile.mkdtemp(dir=paths.private_dir,
+                                    prefix="backupprovision")
+    backup_provision(paths, backupdir)
+    try:
+        ldbs.startTransactions()
 
-    # 5) With all this information let's create a fresh new provision used as
-    # reference
-    message(SIMPLE, "Creating a reference provision")
-    provisiondir = tempfile.mkdtemp(dir=paths.private_dir,
-                                    prefix="referenceprovision")
-    newprovision(names, setup_dir, creds, session, smbconf, provisiondir,
+        # 3) Guess all the needed names (variables in fact) from the current
+        # provision.
+        names = find_provision_key_parameters(ldbs.sam, ldbs.secrets, ldbs.idmap,
+                                                paths, smbconf, lp)
+        # 4)
+        lastProvisionUSNs = get_last_provision_usn(ldbs.sam)
+        if lastProvisionUSNs is not None:
+            v = 0
+            for k in lastProvisionUSNs.keys():
+                for r in lastProvisionUSNs[k]:
+                    v = v + 1
+
+            message(CHANGE,
+                "Find last provision USN, %d invocation(s) for a total of %d ranges" % \
+                            (len(lastProvisionUSNs.keys()), v /2 ))
+
+            if lastProvisionUSNs.get("default") != None:
+                message(CHANGE, "Old style for usn ranges used")
+                lastProvisionUSNs[str(names.invocation)] = lastProvisionUSNs["default"]
+                del lastProvisionUSNs["default"]
+        else:
+            message(SIMPLE, "Your provision lacks provision range information")
+            if confirm("Do you want to run findprovisionusnranges to try to find them ?", False):
+                ldbs.groupedRollback()
+                os.system("%s %s %s %s %s" % (os.path.join(os.path.dirname(sys.argv[0]),
+                                            "findprovisionusnranges"),
+                                        "--storedir",
+                                        paths.private_dir,
+                                        "-s",
+                                        smbconf))
+                message(SIMPLE, "Once you applied/adapted the change(s) please restart the upgradeprovision script")
+                sys.exit(0)
+
+        # Objects will be created with the admin session
+        # (not anymore system session)
+        adm_session = admin_session(lp, str(names.domainsid))
+        # So we reget handle on objects
+        # ldbs = get_ldbs(paths, creds, adm_session, lp)
+        if not opts.fixntacl:
+            if not sanitychecks(ldbs.sam, names):
+                message(SIMPLE, "Sanity checks for the upgrade have failed. "
+                        "Check the messages and correct the errors "
+                        "before rerunning upgradeprovision")
+                ldbs.groupedRollback()
+                sys.exit(1)
+
+            # Let's see provision parameters
+            print_provision_key_parameters(names)
+
+            # 5) With all this information let's create a fresh new provision used as
+            # reference
+            message(SIMPLE, "Creating a reference provision")
+            provisiondir = tempfile.mkdtemp(dir=paths.private_dir,
+                            prefix="referenceprovision")
+            newprovision(names, creds, session, smbconf, provisiondir,
                     provision_logger)
 
-    # TODO
-    # We need to get a list of object which SD is directly computed from
-    # defaultSecurityDescriptor.
-    # This will allow us to know which object we can rebuild the SD in case
-    # of change of the parent's SD or of the defaultSD.
-    # Get file paths of this new provision
-    newpaths = get_paths(param, targetdir=provisiondir)
-    new_ldbs = get_ldbs(newpaths, creds, session, lp)
-    new_ldbs.startTransactions()
-
-    # Populate some associative array to ease the update process
-    # List of attribute which are link and backlink
-    populate_links(new_ldbs.sam, names.schemadn)
-    # List of attribute with ASN DN synthax)
-    populate_dnsyntax(new_ldbs.sam, names.schemadn)
-
-    update_privilege(newpaths.private_dir, paths.private_dir)
-    oem = getOEMInfo(ldbs.sam, names.rootdn)
-    # Do some modification on sam.ldb
-    ldbs.groupedCommit()
-    if re.match(".*alpha((9)|(\d\d+)).*", str(oem)):
-        # Starting from alpha9 we can consider that the structure is quite ok
-        # and that we should do only dela
-        new_ldbs.groupedCommit()
-        delta_update_basesamdb(newpaths, paths, creds, session, lp)
-        ldbs.startTransactions()
-        minUSN = get_max_usn(ldbs.sam, str(names.rootdn)) + 1
-        new_ldbs.startTransactions()
-    else:
-        simple_update_basesamdb(newpaths, paths, names)
-        ldbs = get_ldbs(paths, creds, session, lp)
-        ldbs.startTransactions()
-        removeProvisionUSN(ldbs.sam)
+            # TODO
+            # 6) and 7)
+            # We need to get a list of object which SD is directly computed from
+            # defaultSecurityDescriptor.
+            # This will allow us to know which object we can rebuild the SD in case
+            # of change of the parent's SD or of the defaultSD.
+            # Get file paths of this new provision
+            newpaths = get_paths(param, targetdir=provisiondir)
+            new_ldbs = get_ldbs(newpaths, creds, session, lp)
+            new_ldbs.startTransactions()
+
+            populateNotReplicated(new_ldbs.sam, names.schemadn)
+            # 8) Populate some associative array to ease the update process
+            # List of attribute which are link and backlink
+            populate_links(new_ldbs.sam, names.schemadn)
+            # List of attribute with ASN DN synthax)
+            populate_dnsyntax(new_ldbs.sam, names.schemadn)
+            # 9)
+            update_privilege(newpaths.private_dir, paths.private_dir)
+            # 10)
+            oem = getOEMInfo(ldbs.sam, str(names.rootdn))
+            # Do some modification on sam.ldb
+            ldbs.groupedCommit()
+            new_ldbs.groupedCommit()
+            deltaattr = None
+        # 11)
+            message(GUESS, oem)
+            if oem is None or hasATProvision(ldbs.sam) or re.match(".*alpha((9)|(\d\d+)).*", str(oem)):
+                # 11) A
+                # Starting from alpha9 we can consider that the structure is quite ok
+                # and that we should do only dela
+                deltaattr = delta_update_basesamdb(newpaths.samdb,
+                                paths.samdb,
+                                creds,
+                                session,
+                                lp,
+                                message)
+            else:
+                # 11) B
+                simple_update_basesamdb(newpaths, paths, names)
+                ldbs = get_ldbs(paths, creds, session, lp)
+                removeProvisionUSN(ldbs.sam)
+
+            ldbs.startTransactions()
+            minUSN = int(str(get_max_usn(ldbs.sam, str(names.rootdn)))) + 1
+            new_ldbs.startTransactions()
+
+            # 12)
+            schema = Schema(names.domainsid, schemadn=str(names.schemadn))
+            # We create a closure that will be invoked just before schema reload
+            def schemareloadclosure():
+                basesam = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp,
+                        options=["modules:"])
+                doit = False
+                if deltaattr is not None and len(deltaattr) > 1:
+                    doit = True
+                if doit:
+                    deltaattr.remove("dn")
+                    for att in deltaattr:
+                        if att.lower() == "dn":
+                            continue
+                        if (deltaattr.get(att) is not None
+                            and deltaattr.get(att).flags() != FLAG_MOD_ADD):
+                            doit = False
+                        elif deltaattr.get(att) is None:
+                            doit = False
+                if doit:
+                    message(CHANGE, "Applying delta to @ATTRIBUTES")
+                    deltaattr.dn = ldb.Dn(basesam, "@ATTRIBUTES")
+                    basesam.modify(deltaattr)
+                else:
+                    message(CHANGE, "Not applying delta to @ATTRIBUTES because "
+                        "there is not only add")
+            # 13)
+            if opts.full:
+                if not update_samdb(new_ldbs.sam, ldbs.sam, names, lastProvisionUSNs,
+                        schema, schemareloadclosure):
+                    message(SIMPLE, "Rolling back all changes. Check the cause"
+                            " of the problem")
+                    message(SIMPLE, "Your system is as it was before the upgrade")
+                    ldbs.groupedRollback()
+                    new_ldbs.groupedRollback()
+                    shutil.rmtree(provisiondir)
+                    sys.exit(1)
+            else:
+                # Try to reapply the change also when we do not change the sam
+                # as the delta_upgrade
+                schemareloadclosure()
+                sync_calculated_attributes(ldbs.sam, names)
+                res = ldbs.sam.search(expression="(samaccountname=dns)",
+                        scope=SCOPE_SUBTREE, attrs=["dn"],
+                        controls=["search_options:1:2"])
+                if len(res) > 0:
+                    message(SIMPLE, "You still have the old DNS object for managing "
+                            "dynamic DNS, but you didn't supply --full so "
+                            "a correct update can't be done")
+                    ldbs.groupedRollback()
+                    new_ldbs.groupedRollback()
+                    shutil.rmtree(provisiondir)
+                    sys.exit(1)
+            # 14)
+            update_secrets(new_ldbs.secrets, ldbs.secrets, message)
+            # 14bis)
+            res = ldbs.sam.search(expression="(samaccountname=dns)",
+                        scope=SCOPE_SUBTREE, attrs=["dn"],
+                        controls=["search_options:1:2"])
 
-    schema = Schema(setup_path, names.domainsid, schemadn=str(names.schemadn),
-                    serverdn=str(names.serverdn))
+            if (len(res) == 1):
+                ldbs.sam.delete(res[0]["dn"])
+                res2 = ldbs.secrets.search(expression="(samaccountname=dns)",
+                        scope=SCOPE_SUBTREE, attrs=["dn"])
+                update_dns_account_password(ldbs.sam, ldbs.secrets, names)
+                message(SIMPLE, "IMPORTANT!!! "
+                        "If you were using Dynamic DNS before you need "
+                        "to update your configuration, so that the "
+                        "tkey-gssapi-credential has the following value: "
+                        "DNS/%s.%s" % (names.netbiosname.lower(),
+                            names.realm.lower()))
+            # 15)
+            message(SIMPLE, "Update machine account")
+            update_machine_account_password(ldbs.sam, ldbs.secrets, names)
+
+            dnToRecalculate.sort(dn_sort)
+            # 16) SD should be created with admin but as some previous acl were so wrong
+            # that admin can't modify them we have first to recreate them with the good
+            # form but with system account and then give the ownership to admin ...
+            if str(oem) != "" and not re.match(r'.*alpha(9|\d\d+)', str(oem)):
+                message(SIMPLE, "Fixing very old provision SD")
+                rebuild_sd(ldbs.sam, names)
+
+            # We calculate the max USN before recalculating the SD because we might
+            # touch object that have been modified after a provision and we do not
+            # want that the next upgradeprovision thinks that it has a green light
+            # to modify them
+
+            # 17)
+            maxUSN = get_max_usn(ldbs.sam, str(names.rootdn))
+
+            # 18) We rebuild SD if a we have a list of DN to recalculate or if the
+            # defSDmodified is set.
+            if defSDmodified or len(dnToRecalculate) >0:
+                message(SIMPLE, "Some defaultSecurityDescriptors and/or"
+                                "securityDescriptor have changed, recalculating SD ")
+                ldbs.sam.set_session_info(adm_session)
+                rebuild_sd(ldbs.sam, names)
+
+            # 19)
+            # Now we are quite confident in the recalculate process of the SD, we make
+            # it optional. And we don't do it if there is DN that we must touch
+            # as we are assured that on this DNs we will have differences !
+            # Also the check must be done in a clever way as for the moment we just
+            # compare SDDL
+            if len(dnNotToRecalculate) == 0 and (opts.debugchangesd or opts.debugall):
+                message(CHANGESD, "Checking recalculated SDs")
+                check_updated_sd(new_ldbs.sam, ldbs.sam, names)
+
+            # 20)
+            updateOEMInfo(ldbs.sam, str(names.rootdn))
+            # 21)
+            check_for_DNS(newpaths.private_dir, paths.private_dir)
+            # 22)
+            if lastProvisionUSNs is not None:
+                update_provision_usn(ldbs.sam, minUSN, maxUSN, names.invocation)
+            if opts.full and (names.policyid is None or names.policyid_dc is None):
+                update_policyids(names, ldbs.sam)
+        if opts.full or opts.resetfileacl or opts.fixntacl:
+            try:
+                update_gpo(paths, ldbs.sam, names, lp, message, 1)
+            except ProvisioningError, e:
+                message(ERROR, "The policy for domain controller is missing. "
+                            "You should restart upgradeprovision with --full")
+            except IOError, e:
+                message(ERROR, "Setting ACL not supported on your filesystem")
+        else:
+            try:
+                update_gpo(paths, ldbs.sam, names, lp, message, 0)
+            except ProvisioningError, e:
+                message(ERROR, "The policy for domain controller is missing. "
+                            "You should restart upgradeprovision with --full")
+        if not opts.fixntacl:
+            ldbs.groupedCommit()
+            new_ldbs.groupedCommit()
+            message(SIMPLE, "Upgrade finished!")
+            # remove reference provision now that everything is done !
+            # So we have reindexed first if need when the merged schema was reloaded
+            # (as new attributes could have quick in)
+            # But the second part of the update (when we update existing objects
+            # can also have an influence on indexing as some attribute might have their
+            # searchflag modificated
+            message(SIMPLE, "Reopenning samdb to trigger reindexing if needed "
+                    "after modification")
+            samdb = Ldb(paths.samdb, session_info=session, credentials=creds, lp=lp)
+            message(SIMPLE, "Reindexing finished")
 
-    if opts.full:
-        if not update_samdb(new_ldbs.sam, ldbs.sam, names, lastProvisionUSNs,
-                            schema):
-            message(SIMPLE, "Rollbacking every changes. Check the reason" \
-                            " of the problem")
-            message(SIMPLE, "In any case your system as it was before" \
-                            " the upgrade")
-            ldbs.groupedRollback()
-            new_ldbs.groupedRollback()
             shutil.rmtree(provisiondir)
-            sys.exit(1)
-
-    update_secrets(new_ldbs.secrets, ldbs.secrets)
-    update_machine_account_password(ldbs.sam, ldbs.secrets, names)
-
-    # SD should be created with admin but as some previous acl were so wrong
-    # that admin can't modify them we have first to recreate them with the good
-    # form but with system account and then give the ownership to admin ...
-    if not re.match(r'.*alpha(9|\d\d+)', str(oem)):
-        message(SIMPLE, "Fixing old povision SD")
-        fix_partition_sd(ldbs.sam, names)
-        rebuild_sd(ldbs.sam, names)
-
-    # We calculate the max USN before recalculating the SD because we might
-    # touch object that have been modified after a provision and we do not
-    # want that the next upgradeprovision thinks that it has a green light
-    # to modify them
-
-    maxUSN = get_max_usn(ldbs.sam, str(names.rootdn))
-
-    # We rebuild SD only if defaultSecurityDescriptor is modified
-    # But in fact we should do it also if one object has its SD modified as
-    # child might need rebuild
-    if defSDmodified == 1:
-        message(SIMPLE, "Updating SD")
-        ldbs.sam.set_session_info(adm_session)
-        # Alpha10 was a bit broken still
-        if re.match(r'.*alpha(\d|10)', str(oem)):
-            fix_partition_sd(ldbs.sam, names)
-        rebuild_sd(ldbs.sam, names)
-
-    # Now we are quite confident in the recalculate process of the SD, we make
-    # it optional.
-    # Also the check must be done in a clever way as for the moment we just
-    # compare SDDL
-    if opts.debugchangesd:
-        check_updated_sd(new_ldbs.sam, ldbs.sam, names)
-
-    updateOEMInfo(ldbs.sam, names)
-    check_for_DNS(newpaths.private_dir, paths.private_dir)
-    if lastProvisionUSNs != None:
-        updateProvisionUSN(ldbs.sam, minUSN, maxUSN)
-    ldbs.groupedCommit()
-    new_ldbs.groupedCommit()
-    message(SIMPLE, "Upgrade finished !")
-    # remove reference provision now that everything is done !
-    shutil.rmtree(provisiondir)
+        else:
+            ldbs.groupedRollback()
+            message(SIMPLE, "ACLs fixed !")
+    except StandardError, err:
+        message(ERROR, "A problem occurred while trying to upgrade your "
+                   "provision. A full backup is located at %s" % backupdir)
+        if opts.debugall or opts.debugchange:
+            (typ, val, tb) = sys.exc_info()
+            traceback.print_exception(typ, val, tb)
+        sys.exit(1)