s3upgrade: Add idmap migration, users/groups import
[idra/samba.git] / source4 / scripting / python / samba / upgrade.py
index a118af25265939defb6414ba7d3872a2d608314d..38e6ed87bb019f54d1b139ad2ce16d8aad43b3dc 100644 (file)
@@ -1,22 +1,42 @@
-#!/usr/bin/python
+# backend code for upgrading from Samba3
+# Copyright Jelmer Vernooij 2005-2007
 #
-#      backend code for upgrading from Samba3
-#      Copyright Jelmer Vernooij 2005-2007
-#      Released under the GNU GPL v3 or later
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
 """Support code for upgrading from Samba 3 to Samba 4."""
 
-from provision import findnss, provision, FILL_DRS
+__docformat__ = "restructuredText"
+
 import grp
 import ldb
+import time
 import pwd
-import uuid
-import registry
-from samba import Ldb
-from samba.samdb import SamDB
 
-def import_sam_policy(samldb, samba3_policy, domaindn):
+from samba import Ldb, registry
+from samba.param import LoadParm
+from samba.provision import provision, FILL_FULL
+from samba.samba3 import passdb
+from samba.samba3 import param as s3param
+from samba.dcerpc import lsa
+from samba.dcerpc.security import dom_sid
+from samba import dsdb
+from samba.ndr import ndr_pack
+
+
+def import_sam_policy(samldb, policy, dn):
+    """Import a Samba 3 policy database."""
     samldb.modify_ldif("""
 dn: %s
 changetype: modify
@@ -31,7 +51,7 @@ samba3UserMustLogonToChangePassword: %d
 samba3BadLockoutMinutes: %d
 samba3DisconnectTime: %d
 
-""" % (dn, policy.min_password_length, 
+""" % (dn, policy.min_password_length,
     policy.password_history, policy.minimum_password_age,
     policy.maximum_password_age, policy.lockout_duration,
     policy.reset_count_minutes, policy.user_must_logon_to_change_password,
@@ -40,7 +60,7 @@ samba3DisconnectTime: %d
 
 def import_sam_account(samldb,acc,domaindn,domainsid):
     """Import a Samba 3 SAM account.
-    
+
     :param samldb: Samba 4 SAM Database handle
     :param acc: Samba 3 account
     :param domaindn: Domain DN
@@ -56,7 +76,7 @@ def import_sam_account(samldb,acc,domaindn,domainsid):
 
     if acc.fullname is None:
         acc.fullname = acc.username
-    
+
     assert acc.fullname is not None
     assert acc.nt_username is not None
 
@@ -75,8 +95,8 @@ def import_sam_account(samldb,acc,domaindn,domainsid):
         "samba3Domain": acc.domain,
         "samba3DirDrive": acc.dir_drive,
         "samba3MungedDial": acc.munged_dial,
-        "samba3Homedir": acc.homedir, 
-        "samba3LogonScript": acc.logon_script, 
+        "samba3Homedir": acc.homedir,
+        "samba3LogonScript": acc.logon_script,
         "samba3ProfilePath": acc.profile_path,
         "samba3Workstations": acc.workstations,
         "samba3KickOffTime": str(acc.kickoff_time),
@@ -92,7 +112,7 @@ def import_sam_account(samldb,acc,domaindn,domainsid):
 
 def import_sam_group(samldb, sid, gid, sid_name_use, nt_name, comment, domaindn):
     """Upgrade a SAM group.
-    
+
     :param samldb: SAM database.
     :param gid: Group GID
     :param sid_name_use: SID name use
@@ -106,7 +126,7 @@ def import_sam_group(samldb, sid, gid, sid_name_use, nt_name, comment, domaindn)
 
     if nt_name in ("Domain Guests", "Domain Users", "Domain Admins"):
         return None
-    
+
     if gid == -1:
         gr = grp.getgrnam(nt_name)
     else:
@@ -118,51 +138,152 @@ def import_sam_group(samldb, sid, gid, sid_name_use, nt_name, comment, domaindn)
         unixname = gr.gr_name
 
     assert unixname is not None
-    
+
     samldb.add({
         "dn": "cn=%s,%s" % (nt_name, domaindn),
         "objectClass": ["top", "group"],
         "description": comment,
-        "cn": nt_name, 
+        "cn": nt_name,
         "objectSid": sid,
         "unixName": unixname,
         "samba3SidNameUse": str(sid_name_use)
         })
 
 
-def import_idmap(samdb,samba3_idmap,domaindn):
+def add_idmap_entry(idmapdb, sid, xid, xid_type, logger):
+    """Create idmap entry"""
+
+    # First try to see if we already have this entry
+    found = False
+    try:
+        msg = idmapdb.search(expression='objectSid=%s' % str(sid))
+        if msg.count == 1:
+            found = True
+    except Exception, e:
+        raise e
+
+    if found:
+        print msg.count
+        print dir(msg)
+        try:
+            m = ldb.Message()
+            m.dn = ldb.Dn(idmapdb, msg[0]['dn'])
+            m['xidNumber'] = ldb.MessageElement(str(xid), ldb.FLAG_MOD_REPLACE, 'xidNumber')
+            m['type'] = ldb.MessageElement(xid_type, ldb.FLAG_MOD_REPLACE, 'type')
+            idmapdb.modify(m)
+        except ldb.LdbError, e:
+            logger.warn('Could not modify idmap entry for sid=%s, id=%s, type=%s (%s)',
+                            str(sid), str(xid), xid_type, str(e))
+        except Exception, e:
+            raise e
+    else:
+        try:
+            idmapdb.add({"dn": "CN=%s" % str(sid),
+                        "cn": str(sid),
+                        "objectClass": "sidMap",
+                        "objectSid": ndr_pack(sid),
+                        "type": xid_type,
+                        "xidNumber": str(xid)})
+        except ldb.LdbError, e:
+            logger.warn('Could not add idmap entry for sid=%s, id=%s, type=%s (%s)',
+                            str(sid), str(xid), xid_type, str(e))
+        except Exception, e:
+            raise e
+
+
+def import_idmap(idmapdb, samba3_idmap, logger):
     """Import idmap data.
 
-    :param samdb: SamDB handle.
     :param samba3_idmap: Samba 3 IDMAP database to import from
-    :param domaindn: Domain DN.
     """
-    samdb.add({
-        "dn": domaindn,
-        "userHwm": str(samba3_idmap.get_user_hwm()),
-        "groupHwm": str(samba3_idmap.get_group_hwm())})
 
-    for uid in samba3_idmap.uids():
-        samdb.add({"dn": "SID=%s,%s" % (samba3_idmap.get_user_sid(uid), domaindn),
-                          "SID": samba3_idmap.get_user_sid(uid),
-                          "type": "user",
-                          "unixID": str(uid)})
+    currentxid = max(samba3_idmap.get_user_hwm(), samba3_idmap.get_group_hwm())
+    lowerbound = currentxid
+    # FIXME: upperbound
+
+    m = ldb.Message()
+    m.dn = ldb.Dn(idmapdb, 'CN=CONFIG')
+    m['lowerbound'] = ldb.MessageElement(str(lowerbound), ldb.FLAG_MOD_REPLACE, 'lowerBound')
+    m['xidNumber'] = ldb.MessageElement(str(currentxid), ldb.FLAG_MOD_REPLACE, 'xidNumber')
+    idmapdb.modify(m)
+
+    for id_type, xid in samba3_idmap.ids():
+        if id_type == 'UID':
+            xid_type = 'ID_TYPE_UID'
+        elif id_type == 'GID':
+            xid_type = 'ID_TYPE_GID'
+        else:
+            logger.warn('Wrong type of entry in idmap (%s), Ignoring', id_type)
+            continue
+
+        sid = samba3_idmap.get_sid(xid, id_type)
+        add_idmap_entry(idmapdb, dom_sid(sid), xid, xid_type, logger)
 
-    for gid in samba3_idmap.uids():
-        samdb.add({"dn": "SID=%s,%s" % (samba3_idmap.get_group_sid(gid), domaindn),
-                          "SID": samba3_idmap.get_group_sid(gid),
-                          "type": "group",
-                          "unixID": str(gid)})
+
+def add_group_from_mapping_entry(samdb, groupmap, logger):
+    """Add or modify group from group mapping entry"""
+
+    # First try to see if we already have this entry
+    try:
+        msg = samdb.search(base='<SID=%s>' % str(groupmap.sid), scope=ldb.SCOPE_BASE)
+        found = True
+    except ldb.LdbError, (ecode, emsg):
+        if ecode == ldb.ERR_NO_SUCH_OBJECT:
+            found = False
+        else:
+            raise ldb.LdbError(ecode, emsg)
+    except Exception, e:
+        raise e
+
+    if found:
+        logger.warn('Group already exists sid=%s, groupname=%s existing_groupname=%s, Ignoring.',
+                            str(groupmap.sid), groupmap.nt_name, msg[0]['sAMAccountName'][0])
+    else:
+        if groupmap.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            return
+
+        m = ldb.Message()
+        m.dn = ldb.Dn(samdb, "CN=%s,CN=Users,%s" % (groupmap.nt_name, samdb.get_default_basedn()))
+        m['a01'] = ldb.MessageElement(groupmap.nt_name, ldb.FLAG_MOD_ADD, 'cn')
+        m['a02'] = ldb.MessageElement('group', ldb.FLAG_MOD_ADD, 'objectClass')
+        m['a03'] = ldb.MessageElement(ndr_pack(groupmap.sid), ldb.FLAG_MOD_ADD, 'objectSid')
+        m['a04'] = ldb.MessageElement(groupmap.comment, ldb.FLAG_MOD_ADD, 'description')
+        m['a05'] = ldb.MessageElement(groupmap.nt_name, ldb.FLAG_MOD_ADD, 'sAMAccountName')
+
+        if groupmap.sid_name_use == lsa.SID_NAME_ALIAS:
+            m['a06'] = ldb.MessageElement(str(dsdb.GTYPE_SECURITY_DOMAIN_LOCAL_GROUP), ldb.FLAG_MOD_ADD, 'groupType')
+
+        try:
+            samdb.add(m, controls=["relax:0"])
+        except ldb.LdbError, e:
+            logger.warn('Could not add group name=%s (%s)', groupmap.nt_name, str(e))
+        except Exception, e:
+            raise(e)
+
+
+def add_users_to_group(samdb, group, members):
+    """Add user/member to group/alias"""
+
+    for member_sid in members:
+        m = ldb.Message()
+        m.dn = ldb.Dn(samdb, "<SID=%s" % str(group.sid))
+        m['a01'] = ldb.MessageElement("<SID=%s>" % str(member_sid), ldb.FLAG_MOD_REPLACE, 'member')
+
+        try:
+            samdb.modify(m)
+        except ldb.LdbError, e:
+            logger.warn("Could not add member to group '%s'", groupmap.nt_name)
+        except Exception, e:
+            raise(e)
 
 
 def import_wins(samba4_winsdb, samba3_winsdb):
     """Import settings from a Samba3 WINS database.
-    
+
     :param samba4_winsdb: WINS database to import to
     :param samba3_winsdb: WINS database to import from
     """
     version_id = 0
-    import time
 
     for (name, (ttl, ips, nb_flags)) in samba3_winsdb.items():
         version_id+=1
@@ -201,82 +322,11 @@ def import_wins(samba4_winsdb, samba3_winsdb):
                            "versionID": str(version_id),
                            "address": ips})
 
-    samba4_winsdb.add({"dn": "CN=VERSION",
+    samba4_winsdb.add({"dn": "cn=VERSION",
+                       "cn": "VERSION",
                        "objectClass": "winsMaxVersion",
                        "maxVersion": str(version_id)})
 
-def upgrade_provision(samba3, setup_dir, message, credentials, session_info, lp, paths):
-    oldconf = samba3.get_conf()
-
-    if oldconf.get("domain logons") == "True":
-        serverrole = "domain controller"
-    else:
-        if oldconf.get("security") == "user":
-            serverrole = "standalone"
-        else:
-            serverrole = "member server"
-
-    lp.set("server role", serverrole)
-    domainname = oldconf.get("workgroup")
-    if domainname:
-        domainname = str(domainname)
-    lp.set("workgroup", domainname)
-    realm = oldconf.get("realm")
-    netbiosname = oldconf.get("netbios name")
-
-    secrets_db = samba3.get_secrets_db()
-    
-    if domainname is None:
-        domainname = secrets_db.domains()[0]
-        message("No domain specified in smb.conf file, assuming '%s'" % domainname)
-    
-    if realm is None:
-        realm = domainname.lower()
-        message("No realm specified in smb.conf file, assuming '%s'\n" % realm)
-    lp.set("realm", realm)
-
-    domainguid = secrets_db.get_domain_guid(domainname)
-    domainsid = secrets_db.get_sid(domainname)
-    if domainsid is None:
-        message("Can't find domain secrets for '%s'; using random SID\n" % domainname)
-    
-    if netbiosname is not None:
-        machinepass = secrets_db.get_machine_password(netbiosname)
-    else:
-        machinepass = None
-    
-    domaindn = provision(lp=lp, setup_dir=setup_dir, message=message, 
-                         samdb_fill=FILL_DRS, ldapbackend=None, 
-                         paths=paths, session_info=session_info, credentials=credentials, realm=realm, 
-                         domain=domainname, domainsid=domainsid, domainguid=domainguid, 
-                         machinepass=machinepass, serverrole=serverrole)
-
-    samdb = SamDB(paths.samdb, credentials=credentials, lp=lp, session_info=session_info)
-
-    import_wins(Ldb(paths.winsdb), samba3.get_wins_db())
-
-    # FIXME: import_registry(registry.Registry(), samba3.get_registry())
-
-    # FIXME: import_idmap(samdb,samba3.get_idmap_db(),domaindn)
-    
-    groupdb = samba3.get_groupmapping_db()
-    for sid in groupdb.groupsids():
-        (gid, sid_name_use, nt_name, comment) = groupdb.get_group(sid)
-        # FIXME: import_sam_group(samdb, sid, gid, sid_name_use, nt_name, comment, domaindn)
-
-    # FIXME: Aliases
-
-    passdb = samba3.get_sam_db()
-    for name in passdb:
-        user = passdb[name]
-        #FIXME: import_sam_account(samdb, user, domaindn, domainsid)
-
-    if hasattr(passdb, 'ldap_url'):
-        message("Enabling Samba3 LDAP mappings for SAM database")
-
-        enable_samba3sam(samdb, passdb.ldap_url)
-
-
 def enable_samba3sam(samdb, ldapurl):
     """Enable Samba 3 LDAP URL database.
 
@@ -294,7 +344,7 @@ replace: @LIST
 
 
 smbconf_keep = [
-    "dos charset", 
+    "dos charset",
     "unix charset",
     "display charset",
     "comment",
@@ -338,8 +388,6 @@ smbconf_keep = [
     "write raw",
     "disable netbios",
     "nt status support",
-    "announce version",
-    "announce as",
     "max mux",
     "max xmit",
     "name resolve order",
@@ -391,11 +439,11 @@ def upgrade_smbconf(oldconf,mark):
     """Remove configuration variables not present in Samba4
 
     :param oldconf: Old configuration structure
-    :param mark: Whether removed configuration variables should be 
+    :param mark: Whether removed configuration variables should be
         kept in the new configuration as "samba3:<name>"
     """
     data = oldconf.data()
-    newconf = param_init()
+    newconf = LoadParm()
 
     for s in data:
         for p in data[s]:
@@ -436,3 +484,166 @@ def import_registry(samba4_registry, samba3_regdb):
             key_handle.set_value(value_name, value_type, value_data)
 
 
+def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
+    """Upgrade from samba3 database to samba4 AD database
+    """
+
+    # Read samba3 smb.conf
+    oldconf = s3param.get_context();
+    oldconf.load(smbconf)
+
+    if oldconf.get("domain logons"):
+        serverrole = "domain controller"
+    else:
+        if oldconf.get("security") == "user":
+            serverrole = "standalone"
+        else:
+            serverrole = "member server"
+
+    domainname = oldconf.get("workgroup")
+    realm = oldconf.get("realm")
+    netbiosname = oldconf.get("netbios name")
+
+    # secrets db
+    secrets_db = samba3.get_secrets_db()
+
+    if not domainname:
+        domainname = secrets_db.domains()[0]
+        logger.warning("No domain specified in smb.conf file, assuming '%s'",
+                domainname)
+
+    if not realm:
+        if oldconf.get("domain logons"):
+            logger.warning("No realm specified in smb.conf file and being a DC. That upgrade path doesn't work! Please add a 'realm' directive to your old smb.conf to let us know which one you want to use (generally it's the upcased DNS domainname).")
+            return
+        else:
+            realm = domainname.upper()
+            logger.warning("No realm specified in smb.conf file, assuming '%s'",
+                    realm)
+
+    # Find machine account and password
+    machinepass = None
+    machinerid = None
+    machinesid = None
+    next_rid = 1000
+
+    try:
+        machinepass = secrets_db.get_machine_password(netbiosname)
+    except:
+        pass
+
+    # We must close the direct pytdb database before the C code loads it
+    secrets_db.close()
+
+    passdb.set_secrets_dir(samba3.libdir)
+
+    # Get domain sid
+    try:
+        domainsid = passdb.get_global_sam_sid()
+    except:
+        raise Exception("Can't find domain sid for '%s', Exiting." % domainname)
+
+    # Get machine account, sid, rid
+    try:
+        machineacct = old_passdb.getsampwnam('%s$' % netbiosname)
+        machinesid, machinerid = machineacct.user_sid.split()
+    except:
+        pass
+
+    # Connect to old password backend
+    old_passdb = passdb.PDB(oldconf.get('passdb backend'))
+
+    # Import groups from old passdb backend
+    logger.info("Exporting groups")
+    grouplist = old_passdb.enum_group_mapping()
+    groupmembers = {}
+    for group in grouplist:
+        sid, rid = group.sid.split()
+        if sid == domainsid:
+            if rid >= next_rid:
+               next_rid = rid + 1
+
+        # Get members for each group/alias
+        if group.sid_name_use == lsa.SID_NAME_ALIAS or group.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            members = old_passdb.enum_aliasmem(group.sid)
+        elif group.sid_name_use == lsa.SID_NAME_DOM_GRP:
+            try:
+                members = old_passdb.enum_group_members(group.sid)
+            except:
+                continue
+        else:
+            logger.warn("Ignoring group '%s' with sid_name_use=%d",
+                        group.nt_name, group.sid_name_use)
+            continue
+        groupmembers[group.nt_name] = members
+
+
+    # Import users from old passdb backend
+    logger.info("Exporting users")
+    userlist = old_passdb.search_users(0)
+    userdata = {}
+    uids = {}
+    for entry in userlist:
+        if machinerid and machinerid == entry['rid']:
+            continue
+        username = entry['account_name']
+        if entry['rid'] < 1000:
+            logger.info("  Skipping wellknown rid=%d (for username=%s)", entry['rid'], username)
+            continue
+        if entry['rid'] >= next_rid:
+            next_rid = entry['rid'] + 1
+        
+        userdata[username] = old_passdb.getsampwnam(username)
+        try:
+            uids[username] = old_passdb.sid_to_id(userdata[username].user_sid)[0]
+        except:
+            try:
+                uids[username] = pwd.getpwnam(username).pw_uid
+            except:
+                pass
+
+    logger.info("Next rid = %d", next_rid)
+
+    # Do full provision
+    result = provision(logger, session_info, None,
+                       targetdir=targetdir, realm=realm, domain=domainname,
+                       domainsid=str(domainsid), next_rid=next_rid,
+                       dc_rid=machinerid,
+                       hostname=netbiosname, machinepass=machinepass,
+                       serverrole=serverrole, samdb_fill=FILL_FULL)
+
+    logger.info("Import WINS")
+    import_wins(Ldb(result.paths.winsdb), samba3.get_wins_db())
+
+    new_smbconf = result.lp.configfile
+    newconf = s3param.get_context()
+    newconf.load(new_smbconf)
+
+    # Migrate idmap
+    logger.info("Migrating idmap database")
+    import_idmap(result.idmap, samba3.get_idmap_db(), logger)
+
+    # Connect to samba4 backend
+    new_passdb = passdb.PDB('samba4')
+
+    # Export groups to samba4 backend
+    logger.info("Importing groups")
+    for g in grouplist:
+        # Ignore uninitialized groups (gid = -1)
+        if g.gid != 0xffffffff:
+            add_idmap_entry(result.idmap, g.sid, g.gid, "GID", logger)
+            add_group_from_mapping_entry(result.samdb, g, logger)
+
+    # Export users to samba4 backend
+    logger.info("Importing users")
+    for username in userdata:
+        new_passdb.add_sam_account(userdata[username])
+        if username in uids:
+            add_idmap_entry(result.idmap, userdata[username].user_sid, uids[username], "UID", logger)
+
+    logger.info("Adding users to groups")
+    for g in grouplist:
+        if g.nt_name in groupmembers:
+            add_users_to_group(result.samdb, g, groupmembers[g.nt_name])
+
+    # FIXME: import_registry(registry.Registry(), samba3.get_registry())