s4-s3-upgrade: convert password age policies to the negative NTTIME format
[amitay/samba.git] / source4 / scripting / python / samba / upgrade.py
index 49aee3f94d9407e97ffc337f3db87793864bd446..92ab86b69689aafdd5eedfd0e10d18a2bcf1c1a1 100644 (file)
-#!/usr/bin/python
+# backend code for upgrading from Samba3
+# Copyright Jelmer Vernooij 2005-2007
 #
-#      backend code for upgrading from Samba3
-#      Copyright Jelmer Vernooij 2005-2007
-#      Released under the GNU GPL v3 or later
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 """Support code for upgrading from Samba 3 to Samba 4."""
 
-from provision import findnss
-import provision
+__docformat__ = "restructuredText"
+
 import grp
+import ldb
+import time
 import pwd
-from uuid import uuid4
-from param import default_configuration
 
-def regkey_to_dn(name):
-       dn = "hive=NONE"
+from samba import Ldb, registry
+from samba.param import LoadParm
+from samba.provision import provision, FILL_FULL, ProvisioningError
+from samba.samba3 import passdb
+from samba.samba3 import param as s3param
+from samba.dcerpc import lsa, samr, security
+from samba.dcerpc.security import dom_sid
+from samba import dsdb
+from samba.ndr import ndr_pack
+from samba import unix2nttime
+
+def import_sam_policy(samdb, policy, logger):
+    """Import a Samba 3 policy.
+
+    :param samdb: Samba4 SAM database
+    :param policy: Samba3 account policy
+    :param logger: Logger object
+    """
+
+    # Following entries are used -
+    #    min password length, password history, minimum password age,
+    #    maximum password age, lockout duration
+    #
+    # Following entries are not used -
+    #    reset count minutes, user must logon to change password,
+    #    bad lockout minutes, disconnect time
+
+    m = ldb.Message()
+    m.dn = samdb.get_default_basedn()
+    m['a01'] = ldb.MessageElement(str(unix2nttime(policy['min password length'])), ldb.FLAG_MOD_REPLACE,
+                            'minPwdLength')
+    m['a02'] = ldb.MessageElement(str(policy['password history']), ldb.FLAG_MOD_REPLACE,
+                            'pwdHistoryLength')
+
+    min_pw_age_unix = policy['minimum password age']
+    min_pw_age_nt = 0 - unix2nttime(min_pw_age_unix)
+    m['a03'] = ldb.MessageElement(str(min_pw_age_nt), ldb.FLAG_MOD_REPLACE, 'minPwdAge')
+
+    max_pw_age_unix = policy['maximum password age']
+    if (max_pw_age_unix == 0xFFFFFFFF):
+        max_pw_age_nt = 0
+    else:
+        max_pw_age_nt = unix2nttime(max_pw_age_unix)
+
+    m['a04'] = ldb.MessageElement(str(max_pw_age_nt), ldb.FLAG_MOD_REPLACE,
+                                  'maxPwdAge')
 
-    for el in name.split("/")[1:]:
-        dn = "key=%s," % el + dn
+    lockout_duration_mins = policy['lockout duration']
+    lockout_duration_nt = unix2nttime(lockout_duration_mins * 60)
 
-       return dn
+    m['a05'] = ldb.MessageElement(str(lockout_duration_nt), ldb.FLAG_MOD_REPLACE,
+                                  'lockoutDuration')
 
-# Where prefix is any of:
-# - HKLM
-#   HKU
-#   HKCR
-#   HKPD
-#   HKPT
-#
+    try:
+        samdb.modify(m)
+    except ldb.LdbError, e:
+        logger.warn("Could not set account policy, (%s)", str(e))
+
+
+def add_idmap_entry(idmapdb, sid, xid, xid_type, logger):
+    """Create idmap entry
 
-def upgrade_registry(regdb,prefix,ldb):
-    """Migrate registry contents."""
-    assert regdb is not None:
-       prefix_up = prefix.upper()
-       ldif = []
+    :param idmapdb: Samba4 IDMAP database
+    :param sid: user/group sid
+    :param xid: user/group id
+    :param xid_type: type of id (UID/GID)
+    :param logger: Logger object
+    """
 
-    for rk in regdb.keys:
-               pts = rk.name.split("/")
+    # First try to see if we already have this entry
+    found = False
+    msg = idmapdb.search(expression='objectSid=%s' % str(sid))
+    if msg.count == 1:
+        found = True
 
-               # Only handle selected hive
-        if pts[0].upper() != prefix_up:
-                       continue
+    if found:
+        try:
+            m = ldb.Message()
+            m.dn = msg[0]['dn']
+            m['xidNumber'] = ldb.MessageElement(str(xid), ldb.FLAG_MOD_REPLACE, 'xidNumber')
+            m['type'] = ldb.MessageElement(xid_type, ldb.FLAG_MOD_REPLACE, 'type')
+            idmapdb.modify(m)
+        except ldb.LdbError, e:
+            logger.warn('Could not modify idmap entry for sid=%s, id=%s, type=%s (%s)',
+                            str(sid), str(xid), xid_type, str(e))
+    else:
+        try:
+            idmapdb.add({"dn": "CN=%s" % str(sid),
+                        "cn": str(sid),
+                        "objectClass": "sidMap",
+                        "objectSid": ndr_pack(sid),
+                        "type": xid_type,
+                        "xidNumber": str(xid)})
+        except ldb.LdbError, e:
+            logger.warn('Could not add idmap entry for sid=%s, id=%s, type=%s (%s)',
+                            str(sid), str(xid), xid_type, str(e))
+
+
+def import_idmap(idmapdb, samba3, logger):
+    """Import idmap data.
+
+    :param idmapdb: Samba4 IDMAP database
+    :param samba3_idmap: Samba3 IDMAP database to import from
+    :param logger: Logger object
+    """
+
+    try:
+        samba3_idmap = samba3.get_idmap_db()
+    except IOError as (errno, strerror):
+        logger.warn('Cannot open idmap database, Ignoring: ({0}): {1}'.format(errno, strerror))
+        return
+
+    currentxid = max(samba3_idmap.get_user_hwm(), samba3_idmap.get_group_hwm())
+    lowerbound = currentxid
+    # FIXME: upperbound
+
+    m = ldb.Message()
+    m.dn = ldb.Dn(idmapdb, 'CN=CONFIG')
+    m['lowerbound'] = ldb.MessageElement(str(lowerbound), ldb.FLAG_MOD_REPLACE, 'lowerBound')
+    m['xidNumber'] = ldb.MessageElement(str(currentxid), ldb.FLAG_MOD_REPLACE, 'xidNumber')
+    idmapdb.modify(m)
+
+    for id_type, xid in samba3_idmap.ids():
+        if id_type == 'UID':
+            xid_type = 'ID_TYPE_UID'
+        elif id_type == 'GID':
+            xid_type = 'ID_TYPE_GID'
+        else:
+            logger.warn('Wrong type of entry in idmap (%s), Ignoring', id_type)
+            continue
 
-               keydn = regkey_to_dn(rk.name)
+        sid = samba3_idmap.get_sid(xid, id_type)
+        add_idmap_entry(idmapdb, dom_sid(sid), xid, xid_type, logger)
 
-               pts = rk.name.split("/")
 
-               # Convert key name to dn
-               ldif[rk.name] = """
-dn: %s
-name: %s
+def add_group_from_mapping_entry(samdb, groupmap, logger):
+    """Add or modify group from group mapping entry
 
-""" % (keydn, pts[0])
-               
-        for rv in rk.values:
-                       ldif[rk.name + " (" + rv.name + ")"] = """
-dn: %s,value=%s
-value: %s
-type: %d
-data:: %s""" % (keydn, rv.name, rv.name, rv.type, ldb.encode(rv.data))
+    param samdb: Samba4 SAM database
+    param groupmap: Groupmap entry
+    param logger: Logger object
+    """
 
-       return ldif
+    # First try to see if we already have this entry
+    try:
+        msg = samdb.search(base='<SID=%s>' % str(groupmap.sid), scope=ldb.SCOPE_BASE)
+        found = True
+    except ldb.LdbError, (ecode, emsg):
+        if ecode == ldb.ERR_NO_SUCH_OBJECT:
+            found = False
+        else:
+            raise ldb.LdbError(ecode, emsg)
 
-def upgrade_sam_policy(samba3,dn):
-       ldif = """
-dn: %s
-changetype: modify
-replace: minPwdLength
-minPwdLength: %d
-pwdHistoryLength: %d
-minPwdAge: %d
-maxPwdAge: %d
-lockoutDuration: %d
-samba3ResetCountMinutes: %d
-samba3UserMustLogonToChangePassword: %d
-samba3BadLockoutMinutes: %d
-samba3DisconnectTime: %d
-
-""" % (dn, samba3.policy.min_password_length, 
-       samba3.policy.password_history, samba3.policy.minimum_password_age,
-       samba3.policy.maximum_password_age, samba3.policy.lockout_duration,
-       samba3.policy.reset_count_minutes, samba3.policy.user_must_logon_to_change_password,
-       samba3.policy.bad_lockout_minutes, samba3.policy.disconnect_time)
-       
-       return ldif
-
-def upgrade_sam_account(ldb,acc,domaindn,domainsid):
-    """Upgrade a SAM account."""
-    if acc.nt_username is None or acc.nt_username == "":
-               acc.nt_username = acc.username
-
-    if acc.fullname is None:
-               acc.fullname = pwd.getpwnam(acc.fullname)[4]
-
-       acc.fullname = acc.fullname.split(",")[0]
-
-    if acc.fullname is None:
-               acc.fullname = acc.username
-       
-       assert acc.fullname is not None
-       assert acc.nt_username is not None
-
-       ldif = """dn: cn=%s,%s
-objectClass: top
-objectClass: user
-lastLogon: %d
-lastLogoff: %d
-unixName: %s
-sAMAccountName: %s
-cn: %s
-description: %s
-primaryGroupID: %d
-badPwdcount: %d
-logonCount: %d
-samba3Domain: %s
-samba3DirDrive: %s
-samba3MungedDial: %s
-samba3Homedir: %s
-samba3LogonScript: %s
-samba3ProfilePath: %s
-samba3Workstations: %s
-samba3KickOffTime: %d
-samba3BadPwdTime: %d
-samba3PassLastSetTime: %d
-samba3PassCanChangeTime: %d
-samba3PassMustChangeTime: %d
-objectSid: %s-%d
-lmPwdHash:: %s
-ntPwdHash:: %s
-
-""" % (ldb.dn_escape(acc.fullname), domaindn, acc.logon_time, acc.logoff_time, acc.username, acc.nt_username, acc.nt_username, 
-acc.acct_desc, acc.group_rid, acc.bad_password_count, acc.logon_count,
-acc.domain, acc.dir_drive, acc.munged_dial, acc.homedir, acc.logon_script, 
-acc.profile_path, acc.workstations, acc.kickoff_time, acc.bad_password_time, 
-acc.pass_last_set_time, acc.pass_can_change_time, acc.pass_must_change_time, domainsid, acc.user_rid,
-       ldb.encode(acc.lm_pw), ldb.encode(acc.nt_pw))
-
-       return ldif
-
-def upgrade_sam_group(group,domaindn):
-    """Upgrade a SAM group."""
-       if group.sid_name_use == 5: # Well-known group
-               return None
-
-    if group.nt_name in ("Domain Guests", "Domain Users", "Domain Admins"):
-               return None
-       
-    if group.gid == -1:
-               gr = grp.getgrnam(grp.nt_name)
+    if found:
+        logger.warn('Group already exists sid=%s, groupname=%s existing_groupname=%s, Ignoring.',
+                            str(groupmap.sid), groupmap.nt_name, msg[0]['sAMAccountName'][0])
     else:
-               gr = grp.getgrgid(grp.gid)
+        if groupmap.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            # In a lot of Samba3 databases, aliases are marked as well known groups
+            (group_dom_sid, rid) = group.sid.split()
+            if (group_dom_sid != security.dom_sid(security.SID_BUILTIN)):
+                return
+
+        m = ldb.Message()
+        m.dn = ldb.Dn(samdb, "CN=%s,CN=Users,%s" % (groupmap.nt_name, samdb.get_default_basedn()))
+        m['a01'] = ldb.MessageElement(groupmap.nt_name, ldb.FLAG_MOD_ADD, 'cn')
+        m['a02'] = ldb.MessageElement('group', ldb.FLAG_MOD_ADD, 'objectClass')
+        m['a03'] = ldb.MessageElement(ndr_pack(groupmap.sid), ldb.FLAG_MOD_ADD, 'objectSid')
+        m['a04'] = ldb.MessageElement(groupmap.comment, ldb.FLAG_MOD_ADD, 'description')
+        m['a05'] = ldb.MessageElement(groupmap.nt_name, ldb.FLAG_MOD_ADD, 'sAMAccountName')
+
+        # Fix up incorrect 'well known' groups that are actually builtin (per test above) to be aliases
+        if groupmap.sid_name_use == lsa.SID_NAME_ALIAS or groupmap.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            m['a06'] = ldb.MessageElement(str(dsdb.GTYPE_SECURITY_DOMAIN_LOCAL_GROUP), ldb.FLAG_MOD_ADD, 'groupType')
 
-    if gr is None:
-               group.unixname = "UNKNOWN"
-    else:
-               group.unixname = gr.gr_name
-
-       assert group.unixname is not None
-       
-       ldif = """dn: cn=%s,%s
-objectClass: top
-objectClass: group
-description: %s
-cn: %s
-objectSid: %s
-unixName: %s
-samba3SidNameUse: %d
-""" % (group.nt_name, domaindn, 
-group.comment, group.nt_name, group.sid, group.unixname, group.sid_name_use)
-
-       return ldif
-
-def upgrade_winbind(samba3,domaindn):
-       ldif = """
-               
-dn: dc=none
-userHwm: %d
-groupHwm: %d
-
-""" % (samba3.idmap.user_hwm, samba3.idmap.group_hwm)
-
-    for m in samba3.idmap.mappings:
-               ldif += """
-dn: SID=%s,%s
-SID: %s
-type: %d
-unixID: %d""" % (m.sid, domaindn, m.sid, m.type, m.unix_id)
-       
-       return ldif
-
-def upgrade_wins(samba3):
-       ldif = ""
-       version_id = 0
-
-    for e in samba3.winsentries:
-               now = sys.nttime()
-               ttl = sys.unix2nttime(e.ttl)
-
-               version_id+=1
-
-        numIPs = len(e.ips)
-
-        if e.type == 0x1C:
-                       rType = 0x2
-        elif e.type & 0x80:
-            if numIPs > 1:
-                               rType = 0x2
+        try:
+            samdb.add(m, controls=["relax:0"])
+        except ldb.LdbError, e:
+            logger.warn('Could not add group name=%s (%s)', groupmap.nt_name, str(e))
+
+
+def add_users_to_group(samdb, group, members, logger):
+    """Add user/member to group/alias
+
+    param samdb: Samba4 SAM database
+    param group: Groupmap object
+    param members: List of member SIDs
+    param logger: Logger object
+    """
+    for member_sid in members:
+        m = ldb.Message()
+        m.dn = ldb.Dn(samdb, "<SID=%s>" % str(group.sid))
+        m['a01'] = ldb.MessageElement("<SID=%s>" % str(member_sid), ldb.FLAG_MOD_ADD, 'member')
+
+        try:
+            samdb.modify(m)
+        except ldb.LdbError, (ecode, emsg):
+            if ecode == ldb.ERR_ENTRY_ALREADY_EXISTS:
+                logger.info("skipped re-adding member '%s' to group '%s': %s", member_sid, group.sid, emsg)
+            elif ecode == ldb.ERR_NO_SUCH_OBJECT:
+                raise ProvisioningError("Could not add member '%s' to group '%s' as either group or user record doesn't exist: %s" % (member_sid, group.sid, emsg))
             else:
-                               rType = 0x1
+                raise ProvisioningError("Could not add member '%s' to group '%s': %s" % (member_sid, group.sid, emsg))
+
+
+def import_wins(samba4_winsdb, samba3_winsdb):
+    """Import settings from a Samba3 WINS database.
+
+    :param samba4_winsdb: WINS database to import to
+    :param samba3_winsdb: WINS database to import from
+    """
+    version_id = 0
+
+    for (name, (ttl, ips, nb_flags)) in samba3_winsdb.items():
+        version_id+=1
+
+        type = int(name.split("#", 1)[1], 16)
+
+        if type == 0x1C:
+            rType = 0x2
+        elif type & 0x80:
+            if len(ips) > 1:
+                rType = 0x2
+            else:
+                rType = 0x1
         else:
-            if numIPs > 1:
-                               rType = 0x3
+            if len(ips) > 1:
+                rType = 0x3
             else:
-                               rType = 0x0
+                rType = 0x0
 
-        if ttl > now:
-                       rState = 0x0 # active
+        if ttl > time.time():
+            rState = 0x0 # active
         else:
-                       rState = 0x1 # released
-
-               nType = ((e.nb_flags & 0x60)>>5)
-
-               ldif += """
-dn: name=%s,type=0x%02X
-type: 0x%02X
-name: %s
-objectClass: winsRecord
-recordType: %u
-recordState: %u
-nodeType: %u
-isStatic: 0
-expireTime: %s
-versionID: %llu
-""" % (e.name, e.type, e.type, e.name, 
-   rType, rState, nType, 
-   ldaptime(ttl), version_id)
-
-        for ip in e.ips:
-                       ldif += "address: %s\n" % ip
-
-       ldif += """
-dn: CN=VERSION
-objectClass: winsMaxVersion
-maxVersion: %llu
-""" % version_id
-
-       return ldif
-
-def upgrade_provision(lp, samba3):
-       subobj = Object()
-
-       domainname = samba3.configuration.get("workgroup")
-       
-    if domainname is None:
-               domainname = samba3.secrets.domains[0].name
-               print "No domain specified in smb.conf file, assuming '%s'\n" % domainname
-       
-       domsec = samba3.find_domainsecrets(domainname)
-       hostsec = samba3.find_domainsecrets(hostname())
-       realm = samba3.configuration.get("realm")
-
-    if realm is None:
-               realm = domainname
-               print "No realm specified in smb.conf file, assuming '%s'\n" % realm
-       random_init(local)
-
-       subobj.realm        = realm
-       subobj.domain       = domainname
-       subobj.hostname     = hostname()
-
-       assert subobj.realm is not None
-       assert subobj.domain is not None
-       assert subobj.hostname is not None
-
-       subobj.HOSTIP       = hostip()
-    if domsec is not None:
-               subobj.DOMAINGUID   = domsec.guid
-               subobj.DOMAINSID    = domsec.sid
-    else:
-               print "Can't find domain secrets for '%s'; using random SID and GUID\n" % domainname
-               subobj.DOMAINGUID = uuid4()
-               subobj.DOMAINSID = randsid()
-       
-    if hostsec:
-               subobj.HOSTGUID     = hostsec.guid
-    else:
-               subobj.HOSTGUID = uuid4()
-       subobj.invocationid = uuid4()
-       subobj.krbtgtpass   = randpass(12)
-       subobj.machinepass  = randpass(12)
-       subobj.adminpass    = randpass(12)
-       subobj.datestring   = datestring()
-       subobj.root         = findnss(pwd.getpwnam, "root")[4]
-       subobj.nobody       = findnss(pwd.getpwnam, "nobody")[4]
-       subobj.nogroup      = findnss(grp.getgrnam, "nogroup", "nobody")[2]
-       subobj.wheel        = findnss(grp.getgrnam, "wheel", "root")[2]
-       subobj.users        = findnss(grp.getgrnam, "users", "guest", "other")[2]
-       subobj.dnsdomain    = subobj.realm.lower()
-       subobj.dnsname      = "%s.%s" % (subobj.hostname.lower(), subobj.dnsdomain)
-       subobj.basedn       = "DC=" + ",DC=".join(subobj.realm.split("."))
-       rdn_list = subobj.dnsdomain.split(".")
-       subobj.domaindn     = "DC=" + ",DC=".join(rdn_list)
-       subobj.domaindn_ldb = "users.ldb"
-       subobj.rootdn       = subobj.domaindn
-
-       modules_list        = ["rootdse",
-                                       "kludge_acl",
-                                       "paged_results",
-                                       "server_sort",
-                                       "extended_dn",
-                                       "asq",
-                                       "samldb",
-                                       "password_hash",
-                                       "operational",
-                                       "objectclass",
-                                       "rdn_name",
-                                       "show_deleted",
-                                       "partition"]
-       subobj.modules_list = ",".join(modules_list)
-
-       return subobj
+            rState = 0x1 # released
+
+        nType = ((nb_flags & 0x60)>>5)
+
+        samba4_winsdb.add({"dn": "name=%s,type=0x%s" % tuple(name.split("#")),
+                           "type": name.split("#")[1],
+                           "name": name.split("#")[0],
+                           "objectClass": "winsRecord",
+                           "recordType": str(rType),
+                           "recordState": str(rState),
+                           "nodeType": str(nType),
+                           "expireTime": ldb.timestring(ttl),
+                           "isStatic": "0",
+                           "versionID": str(version_id),
+                           "address": ips})
+
+    samba4_winsdb.add({"dn": "cn=VERSION",
+                       "cn": "VERSION",
+                       "objectClass": "winsMaxVersion",
+                       "maxVersion": str(version_id)})
+
+def enable_samba3sam(samdb, ldapurl):
+    """Enable Samba 3 LDAP URL database.
+
+    :param samdb: SAM Database.
+    :param ldapurl: Samba 3 LDAP URL
+    """
+    samdb.modify_ldif("""
+dn: @MODULES
+changetype: modify
+replace: @LIST
+@LIST: samldb,operational,objectguid,rdn_name,samba3sam
+""")
+
+    samdb.add({"dn": "@MAP=samba3sam", "@MAP_URL": ldapurl})
+
 
 smbconf_keep = [
-       "dos charset", 
-       "unix charset",
-       "display charset",
-       "comment",
-       "path",
-       "directory",
-       "workgroup",
-       "realm",
-       "netbios name",
-       "netbios aliases",
-       "netbios scope",
-       "server string",
-       "interfaces",
-       "bind interfaces only",
-       "security",
-       "auth methods",
-       "encrypt passwords",
-       "null passwords",
-       "obey pam restrictions",
-       "password server",
-       "smb passwd file",
-       "private dir",
-       "passwd chat",
-       "password level",
-       "lanman auth",
-       "ntlm auth",
-       "client NTLMv2 auth",
-       "client lanman auth",
-       "client plaintext auth",
-       "read only",
-       "hosts allow",
-       "hosts deny",
-       "log level",
-       "debuglevel",
-       "log file",
-       "smb ports",
-       "large readwrite",
-       "max protocol",
-       "min protocol",
-       "unicode",
-       "read raw",
-       "write raw",
-       "disable netbios",
-       "nt status support",
-       "announce version",
-       "announce as",
-       "max mux",
-       "max xmit",
-       "name resolve order",
-       "max wins ttl",
-       "min wins ttl",
-       "time server",
-       "unix extensions",
-       "use spnego",
-       "server signing",
-       "client signing",
-       "max connections",
-       "paranoid server security",
-       "socket options",
-       "strict sync",
-       "max print jobs",
-       "printable",
-       "print ok",
-       "printer name",
-       "printer",
-       "map system",
-       "map hidden",
-       "map archive",
-       "preferred master",
-       "prefered master",
-       "local master",
-       "browseable",
-       "browsable",
-       "wins server",
-       "wins support",
-       "csc policy",
-       "strict locking",
-       "preload",
-       "auto services",
-       "lock dir",
-       "lock directory",
-       "pid directory",
-       "socket address",
-       "copy",
-       "include",
-       "available",
-       "volume",
-       "fstype",
-       "panic action",
-       "msdfs root",
-       "host msdfs",
-       "winbind separator"]
+    "dos charset",
+    "unix charset",
+    "display charset",
+    "comment",
+    "path",
+    "directory",
+    "workgroup",
+    "realm",
+    "netbios name",
+    "netbios aliases",
+    "netbios scope",
+    "server string",
+    "interfaces",
+    "bind interfaces only",
+    "security",
+    "auth methods",
+    "encrypt passwords",
+    "null passwords",
+    "obey pam restrictions",
+    "password server",
+    "smb passwd file",
+    "private dir",
+    "passwd chat",
+    "password level",
+    "lanman auth",
+    "ntlm auth",
+    "client NTLMv2 auth",
+    "client lanman auth",
+    "client plaintext auth",
+    "read only",
+    "hosts allow",
+    "hosts deny",
+    "log level",
+    "debuglevel",
+    "log file",
+    "smb ports",
+    "large readwrite",
+    "max protocol",
+    "min protocol",
+    "unicode",
+    "read raw",
+    "write raw",
+    "disable netbios",
+    "nt status support",
+    "max mux",
+    "max xmit",
+    "name resolve order",
+    "max wins ttl",
+    "min wins ttl",
+    "time server",
+    "unix extensions",
+    "use spnego",
+    "server signing",
+    "client signing",
+    "max connections",
+    "paranoid server security",
+    "socket options",
+    "strict sync",
+    "max print jobs",
+    "printable",
+    "print ok",
+    "printer name",
+    "printer",
+    "map system",
+    "map hidden",
+    "map archive",
+    "preferred master",
+    "prefered master",
+    "local master",
+    "browseable",
+    "browsable",
+    "wins server",
+    "wins support",
+    "csc policy",
+    "strict locking",
+    "preload",
+    "auto services",
+    "lock dir",
+    "lock directory",
+    "pid directory",
+    "socket address",
+    "copy",
+    "include",
+    "available",
+    "volume",
+    "fstype",
+    "panic action",
+    "msdfs root",
+    "host msdfs",
+    "winbind separator"]
 
-#
-#   Remove configuration variables not present in Samba4
-#      oldconf: Old configuration structure
-#      mark: Whether removed configuration variables should be 
-#              kept in the new configuration as "samba3:<name>"
 def upgrade_smbconf(oldconf,mark):
-       data = oldconf.data()
-       newconf = param_init()
-
-       for (s in data) {
-               for (p in data[s]) {
-                       keep = False
-                       for (k in smbconf_keep) { 
+    """Remove configuration variables not present in Samba4
+
+    :param oldconf: Old configuration structure
+    :param mark: Whether removed configuration variables should be
+        kept in the new configuration as "samba3:<name>"
+    """
+    data = oldconf.data()
+    newconf = LoadParm()
+
+    for s in data:
+        for p in data[s]:
+            keep = False
+            for k in smbconf_keep:
                 if smbconf_keep[k] == p:
-                                       keep = True
-                                       break
-                       }
+                    keep = True
+                    break
 
             if keep:
-                               newconf.set(s, p, oldconf.get(s, p))
+                newconf.set(s, p, oldconf.get(s, p))
             elif mark:
-                               newconf.set(s, "samba3:"+p, oldconf.get(s,p))
-               }
-       }
+                newconf.set(s, "samba3:"+p, oldconf.get(s,p))
+
+    return newconf
+
+SAMBA3_PREDEF_NAMES = {
+        'HKLM': registry.HKEY_LOCAL_MACHINE,
+}
+
+def import_registry(samba4_registry, samba3_regdb):
+    """Import a Samba 3 registry database into the Samba 4 registry.
+
+    :param samba4_registry: Samba 4 registry handle.
+    :param samba3_regdb: Samba 3 registry database handle.
+    """
+    def ensure_key_exists(keypath):
+        (predef_name, keypath) = keypath.split("/", 1)
+        predef_id = SAMBA3_PREDEF_NAMES[predef_name]
+        keypath = keypath.replace("/", "\\")
+        return samba4_registry.create_key(predef_id, keypath)
+
+    for key in samba3_regdb.keys():
+        key_handle = ensure_key_exists(key)
+        for subkey in samba3_regdb.subkeys(key):
+            ensure_key_exists(subkey)
+        for (value_name, (value_type, value_data)) in samba3_regdb.values(key).items():
+            key_handle.set_value(value_name, value_type, value_data)
+
+
+def upgrade_from_samba3(samba3, logger, targetdir, session_info=None, useeadb=False):
+    """Upgrade from samba3 database to samba4 AD database
+
+    :param samba3: samba3 object
+    :param logger: Logger object
+    :param targetdir: samba4 database directory
+    :param session_info: Session information
+    """
 
-    if oldconf.get("domain logons") == "True":
-               newconf.set("server role", "domain controller")
+    if samba3.lp.get("domain logons"):
+        serverrole = "domain controller"
     else:
-        if oldconf.get("security") == "user":
-                       newconf.set("server role", "standalone")
+        if samba3.lp.get("security") == "user":
+            serverrole = "standalone"
         else:
-                       newconf.set("server role", "member server")
-
-       return newconf
-
-def upgrade(subobj, samba3, message, paths, session_info, credentials):
-       ret = 0
-       lp = loadparm_init()
-       samdb = Ldb(paths.samdb, session_info=session_info, credentials=credentials)
-
-       message("Writing configuration")
-       newconf = upgrade_smbconf(samba3.configuration,True)
-       newconf.save(paths.smbconf)
-
-       message("Importing account policies")
-       ldif = upgrade_sam_policy(samba3,subobj.BASEDN)
-       samdb.modify(ldif)
-       regdb = Ldb(paths.hklm)
-
-       regdb.modify("
-dn: value=RefusePasswordChange,key=Parameters,key=Netlogon,key=Services,key=CurrentControlSet,key=System,HIVE=NONE
-replace: type
-type: 4
-replace: data
-data: %d
-" % samba3.policy.refuse_machine_password_change)
-
-       message("Importing users")
-    for account in samba3.samaccounts:
-               msg = "... " + account.username
-               ldif = upgrade_sam_account(samdb, accounts,subobj.BASEDN,subobj.DOMAINSID)
-        try:
-            samdb.add(ldif)
-        except LdbError, e:
-            # FIXME: Ignore 'Record exists' errors
-                       msg += "... error: " + str(e)
-                       ret += 1; 
-               message(msg)
-
-       message("Importing groups")
-    for mapping in samba3.groupmappings:
-               msg = "... " + mapping.nt_name
-               ldif = upgrade_sam_group(mapping, subobj.BASEDN)
-        if ldif is not None:
-            try:
-                           samdb.add(ldif)
-            except LdbError, e:
-                # FIXME: Ignore 'Record exists' errors
-                               msg += "... error: " + str(e)
-                               ret += 1
-               message(msg)
-
-       message("Importing registry data")
-    for hive in ["hkcr","hkcu","hklm","hkpd","hku","hkpt"]:
-               message("... " + hive)
-               regdb = Ldb(paths[hive])
-               ldif = upgrade_registry(samba3.registry, hive, regdb)
-               for (var j in ldif) {
-                       var msg = "... ... " + j
-            try:
-                regdb.add(ldif[j])
-            except LdbError, e:
-                # FIXME: Ignore 'Record exists' errors
-                               msg += "... error: " + str(e)
-                               ret += 1
-                       message(msg)
-
-       message("Importing WINS data")
-       winsdb = Ldb(paths.winsdb)
-       ldb_erase(winsdb)
-
-       ldif = upgrade_wins(samba3)
-       winsdb.add(ldif)
-
-       # figure out ldapurl, if applicable
-       ldapurl = None
-       pdb = samba3.configuration.get_list("passdb backend")
-    if pdb is not None:
-        for backend in pdb:
-            if len(backend) >= 7 and backend[0:7] == "ldapsam":
-                ldapurl = backend[7:]
-
-       # URL was not specified in passdb backend but ldap /is/ used
-    if ldapurl == "":
-           ldapurl = "ldap://%s" % samba3.configuration.get("ldap server")
-
-       # Enable samba3sam module if original passdb backend was ldap
-    if ldapurl is not None:
-               message("Enabling Samba3 LDAP mappings for SAM database")
-
-               samdb.modify("""
-dn: @MODULES
-changetype: modify
-replace: @LIST
-@LIST: samldb,operational,objectguid,rdn_name,samba3sam
-""")
+            serverrole = "member server"
 
-               samdb.add("""
-dn: @MAP=samba3sam
-@MAP_URL: %s""", ldapurl))
+    domainname = samba3.lp.get("workgroup")
+    realm = samba3.lp.get("realm")
+    netbiosname = samba3.lp.get("netbios name")
 
-       return ret
+    # secrets db
+    secrets_db = samba3.get_secrets_db()
 
-def upgrade_verify(subobj, samba3, paths, message):
-       message("Verifying account policies")
+    if not domainname:
+        domainname = secrets_db.domains()[0]
+        logger.warning("No workgroup specified in smb.conf file, assuming '%s'",
+                domainname)
 
-       samldb = Ldb(paths.samdb)
-
-    for account in samba3.samaccounts:
-               msg = samldb.search("(&(sAMAccountName=" + account.nt_username + ")(objectclass=user))")
-               assert(len(msg) >= 1)
-       
-       # FIXME
+    if not realm:
+        if serverrole == "domain controller":
+            raise ProvisioningError("No realm specified in smb.conf file and being a DC. That upgrade path doesn't work! Please add a 'realm' directive to your old smb.conf to let us know which one you want to use (it is the DNS name of the AD domain you wish to create.")
+        else:
+            realm = domainname.upper()
+            logger.warning("No realm specified in smb.conf file, assuming '%s'",
+                    realm)
+
+    # Find machine account and password
+    machinepass = None
+    machinerid = None
+    machinesid = None
+    next_rid = 1000
+
+    try:
+        machinepass = secrets_db.get_machine_password(netbiosname)
+    except:
+        pass
+
+    # We must close the direct pytdb database before the C code loads it
+    secrets_db.close()
+
+    # Connect to old password backend
+    passdb.set_secrets_dir(samba3.lp.get("private dir"))
+    s3db = samba3.get_sam_db()
+
+    # Get domain sid
+    try:
+        domainsid = passdb.get_global_sam_sid()
+    except passdb.error:
+        raise Exception("Can't find domain sid for '%s', Exiting." % domainname)
+
+    # Get machine account, sid, rid
+    try:
+        machineacct = s3db.getsampwnam('%s$' % netbiosname)
+        machinesid, machinerid = machineacct.user_sid.split()
+    except:
+        pass
+
+    # Export account policy
+    logger.info("Exporting account policy")
+    policy = s3db.get_account_policy()
+
+    # Export groups from old passdb backend
+    logger.info("Exporting groups")
+    grouplist = s3db.enum_group_mapping()
+    groupmembers = {}
+    for group in grouplist:
+        sid, rid = group.sid.split()
+        if sid == domainsid:
+            if rid >= next_rid:
+               next_rid = rid + 1
+
+        # Get members for each group/alias
+        if group.sid_name_use == lsa.SID_NAME_ALIAS:
+            members = s3db.enum_aliasmem(group.sid)
+        elif group.sid_name_use == lsa.SID_NAME_DOM_GRP:
+            try:
+                members = s3db.enum_group_members(group.sid)
+            except:
+                continue
+            groupmembers[group.nt_name] = members
+        elif group.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            (group_dom_sid, rid) = group.sid.split()
+            if (group_dom_sid != security.dom_sid(security.SID_BUILTIN)):
+                logger.warn("Ignoring 'well known' group '%s' (should already be in AD, and have no members)",
+                            group.nt_name)
+                continue
+            # A number of buggy databases mix up well known groups and aliases.
+            members = s3db.enum_aliasmem(group.sid)
+        else:
+            logger.warn("Ignoring group '%s' with sid_name_use=%d",
+                        group.nt_name, group.sid_name_use)
+            continue
+
+
+    # Export users from old passdb backend
+    logger.info("Exporting users")
+    userlist = s3db.search_users(0)
+    userdata = {}
+    uids = {}
+    admin_user = None
+    for entry in userlist:
+        if machinerid and machinerid == entry['rid']:
+            continue
+        username = entry['account_name']
+        if entry['rid'] < 1000:
+            logger.info("  Skipping wellknown rid=%d (for username=%s)", entry['rid'], username)
+            continue
+        if entry['rid'] >= next_rid:
+            next_rid = entry['rid'] + 1
+
+        user = s3db.getsampwnam(username)
+        acct_type = (user.acct_ctrl & (samr.ACB_NORMAL|samr.ACB_WSTRUST|samr.ACB_SVRTRUST|samr.ACB_DOMTRUST))
+        if (acct_type == samr.ACB_NORMAL or acct_type == samr.ACB_WSTRUST or acct_type == samr.ACB_SVRTRUST):
+            pass
+        elif acct_type == samr.ACB_DOMTRUST:
+            logger.warn("  Skipping inter-domain trust from domain %s, this trust must be re-created as an AD trust" % username[:-1])
+            continue
+        elif acct_type == (samr.ACB_NORMAL|samr.ACB_WSTRUST) and username[-1] == '$':
+            logger.warn("  Fixing account %s which had both ACB_NORMAL (U) and ACB_WSTRUST (W) set.  Account will be marked as ACB_WSTRUST (W), i.e. as a domain member" % username)
+            user.acct_ctrl = (user.acct_ctrl & ~samr.ACB_NORMAL)
+        else:
+            raise ProvisioningError("""Failed to upgrade due to invalid account %s, account control flags 0x%08X must have exactly one of
+ACB_NORMAL (N, 0x%08X), ACB_WSTRUST (W 0x%08X), ACB_SVRTRUST (S 0x%08X) or ACB_DOMTRUST (D 0x%08X).
+
+Please fix this account before attempting to upgrade again
+"""
+                                    % (user.acct_flags, username,
+                                       samr.ACB_NORMAL, samr.ACB_WSTRUST, samr.ACB_SVRTRUST, samr.ACB_DOMTRUST))
+        
+        userdata[username] = user
+        try:
+            uids[username] = s3db.sid_to_id(user.user_sid)[0]
+        except:
+            try:
+                uids[username] = pwd.getpwnam(username).pw_uid
+            except:
+                pass
+
+        if not admin_user and username.lower() == 'root':
+            admin_user = username
+        if username.lower() == 'administrator':
+            admin_user = username
+
+    logger.info("Next rid = %d", next_rid)
+
+    # Do full provision
+    result = provision(logger, session_info, None,
+                       targetdir=targetdir, realm=realm, domain=domainname,
+                       domainsid=str(domainsid), next_rid=next_rid,
+                       dc_rid=machinerid,
+                       hostname=netbiosname, machinepass=machinepass,
+                       serverrole=serverrole, samdb_fill=FILL_FULL,
+                       useeadb=useeadb)
+
+    # Import WINS database
+    logger.info("Importing WINS database")
+    import_wins(Ldb(result.paths.winsdb), samba3.get_wins_db())
+
+    # Set Account policy
+    logger.info("Importing Account policy")
+    import_sam_policy(result.samdb, policy, logger)
+
+    # Migrate IDMAP database
+    logger.info("Importing idmap database")
+    import_idmap(result.idmap, samba3, logger)
+
+    # Set the s3 context for samba4 configuration
+    new_lp_ctx = s3param.get_context()
+    new_lp_ctx.load(result.lp.configfile)
+    new_lp_ctx.set("private dir", result.lp.get("private dir"))
+    new_lp_ctx.set("state directory", result.lp.get("state directory"))
+    new_lp_ctx.set("lock directory", result.lp.get("lock directory"))
+
+    # Connect to samba4 backend
+    s4_passdb = passdb.PDB(new_lp_ctx.get("passdb backend"))
+
+    # Export groups to samba4 backend
+    logger.info("Importing groups")
+    for g in grouplist:
+        # Ignore uninitialized groups (gid = -1)
+        if g.gid != 0xffffffff:
+            add_idmap_entry(result.idmap, g.sid, g.gid, "GID", logger)
+            add_group_from_mapping_entry(result.samdb, g, logger)
+
+    # Export users to samba4 backend
+    logger.info("Importing users")
+    for username in userdata:
+        if username.lower() == 'administrator' or username.lower() == 'root':
+            continue
+        s4_passdb.add_sam_account(userdata[username])
+        if username in uids:
+            add_idmap_entry(result.idmap, userdata[username].user_sid, uids[username], "UID", logger)
+
+    logger.info("Adding users to groups")
+    for g in grouplist:
+        if g.nt_name in groupmembers:
+            add_users_to_group(result.samdb, g, groupmembers[g.nt_name], logger)
+
+    # Set password for administrator
+    if admin_user:
+        logger.info("Setting password for administrator")
+        admin_userdata = s4_passdb.getsampwnam("administrator")
+        admin_userdata.nt_passwd = userdata[admin_user].nt_passwd
+        if userdata[admin_user].lanman_passwd:
+            admin_userdata.lanman_passwd = userdata[admin_user].lanman_passwd
+        admin_userdata.pass_last_set_time = userdata[admin_user].pass_last_set_time
+        if userdata[admin_user].pw_history:
+            admin_userdata.pw_history = userdata[admin_user].pw_history
+        s4_passdb.update_sam_account(admin_userdata)
+        logger.info("Administrator password has been set to password of user '%s'", admin_user)
+
+    # FIXME: import_registry(registry.Registry(), samba3.get_registry())
+    # FIXME: shares