s4-s3-upgrade Handle expected errors, error out on unexpected ones
[amitay/samba.git] / source4 / scripting / python / samba / upgrade.py
index 579c9e6f117ebb1ad3a25473757318806af148c7..2bb5ccccbe16aa8304b56422ecfacf2c2b76e38f 100644 (file)
@@ -26,156 +26,76 @@ import pwd
 
 from samba import Ldb, registry
 from samba.param import LoadParm
-from samba.provision import provision, FILL_FULL
+from samba.provision import provision, FILL_FULL, ProvisioningError
 from samba.samba3 import passdb
 from samba.samba3 import param as s3param
-from samba.dcerpc import lsa
+from samba.dcerpc import lsa, samr, security
 from samba.dcerpc.security import dom_sid
 from samba import dsdb
 from samba.ndr import ndr_pack
 
 
-def import_sam_policy(samldb, policy, dn):
-    """Import a Samba 3 policy database."""
-    samldb.modify_ldif("""
-dn: %s
-changetype: modify
-replace: minPwdLength
-minPwdLength: %d
-pwdHistoryLength: %d
-minPwdAge: %d
-maxPwdAge: %d
-lockoutDuration: %d
-samba3ResetCountMinutes: %d
-samba3UserMustLogonToChangePassword: %d
-samba3BadLockoutMinutes: %d
-samba3DisconnectTime: %d
-
-""" % (dn, policy.min_password_length,
-    policy.password_history, policy.minimum_password_age,
-    policy.maximum_password_age, policy.lockout_duration,
-    policy.reset_count_minutes, policy.user_must_logon_to_change_password,
-    policy.bad_lockout_minutes, policy.disconnect_time))
-
-
-def import_sam_account(samldb,acc,domaindn,domainsid):
-    """Import a Samba 3 SAM account.
-
-    :param samldb: Samba 4 SAM Database handle
-    :param acc: Samba 3 account
-    :param domaindn: Domain DN
-    :param domainsid: Domain SID."""
-    if acc.nt_username is None or acc.nt_username == "":
-        acc.nt_username = acc.username
-
-    if acc.fullname is None:
-        try:
-            acc.fullname = pwd.getpwnam(acc.username)[4].split(",")[0]
-        except KeyError:
-            pass
+def import_sam_policy(samdb, policy, logger):
+    """Import a Samba 3 policy.
 
-    if acc.fullname is None:
-        acc.fullname = acc.username
-
-    assert acc.fullname is not None
-    assert acc.nt_username is not None
-
-    samldb.add({
-        "dn": "cn=%s,%s" % (acc.fullname, domaindn),
-        "objectClass": ["top", "user"],
-        "lastLogon": str(acc.logon_time),
-        "lastLogoff": str(acc.logoff_time),
-        "unixName": acc.username,
-        "sAMAccountName": acc.nt_username,
-        "cn": acc.nt_username,
-        "description": acc.acct_desc,
-        "primaryGroupID": str(acc.group_rid),
-        "badPwdcount": str(acc.bad_password_count),
-        "logonCount": str(acc.logon_count),
-        "samba3Domain": acc.domain,
-        "samba3DirDrive": acc.dir_drive,
-        "samba3MungedDial": acc.munged_dial,
-        "samba3Homedir": acc.homedir,
-        "samba3LogonScript": acc.logon_script,
-        "samba3ProfilePath": acc.profile_path,
-        "samba3Workstations": acc.workstations,
-        "samba3KickOffTime": str(acc.kickoff_time),
-        "samba3BadPwdTime": str(acc.bad_password_time),
-        "samba3PassLastSetTime": str(acc.pass_last_set_time),
-        "samba3PassCanChangeTime": str(acc.pass_can_change_time),
-        "samba3PassMustChangeTime": str(acc.pass_must_change_time),
-        "objectSid": "%s-%d" % (domainsid, acc.user_rid),
-        "lmPwdHash:": acc.lm_password,
-        "ntPwdHash:": acc.nt_password,
-        })
-
-
-def import_sam_group(samldb, sid, gid, sid_name_use, nt_name, comment, domaindn):
-    """Upgrade a SAM group.
-
-    :param samldb: SAM database.
-    :param gid: Group GID
-    :param sid_name_use: SID name use
-    :param nt_name: NT Group Name
-    :param comment: NT Group Comment
-    :param domaindn: Domain DN
+    :param samdb: Samba4 SAM database
+    :param policy: Samba3 account policy
+    :param logger: Logger object
     """
 
-    if sid_name_use == 5: # Well-known group
-        return None
+    # Following entries are used -
+    #    min password length, password history, minimum password age,
+    #    maximum password age, lockout duration
+    #
+    # Following entries are not used -
+    #    reset count minutes, user must logon to change password,
+    #    bad lockout minutes, disconnect time
 
-    if nt_name in ("Domain Guests", "Domain Users", "Domain Admins"):
-        return None
-
-    if gid == -1:
-        gr = grp.getgrnam(nt_name)
-    else:
-        gr = grp.getgrgid(gid)
-
-    if gr is None:
-        unixname = "UNKNOWN"
-    else:
-        unixname = gr.gr_name
-
-    assert unixname is not None
+    m = ldb.Message()
+    m.dn = samdb.get_default_basedn()
+    m['a01'] = ldb.MessageElement(str(policy['min password length']), ldb.FLAG_MOD_REPLACE,
+                            'minPwdLength')
+    m['a02'] = ldb.MessageElement(str(policy['password history']), ldb.FLAG_MOD_REPLACE,
+                            'pwdHistoryLength')
+    m['a03'] = ldb.MessageElement(str(policy['minimum password age']), ldb.FLAG_MOD_REPLACE,
+                            'minPwdAge')
+    m['a04'] = ldb.MessageElement(str(policy['maximum password age']), ldb.FLAG_MOD_REPLACE,
+                            'maxPwdAge')
+    m['a05'] = ldb.MessageElement(str(policy['lockout duration']), ldb.FLAG_MOD_REPLACE,
+                            'lockoutDuration')
 
-    samldb.add({
-        "dn": "cn=%s,%s" % (nt_name, domaindn),
-        "objectClass": ["top", "group"],
-        "description": comment,
-        "cn": nt_name,
-        "objectSid": sid,
-        "unixName": unixname,
-        "samba3SidNameUse": str(sid_name_use)
-        })
+    try:
+        samdb.modify(m)
+    except ldb.LdbError, e:
+        logger.warn("Could not set account policy, (%s)", str(e))
 
 
 def add_idmap_entry(idmapdb, sid, xid, xid_type, logger):
-    """Create idmap entry"""
+    """Create idmap entry
+
+    :param idmapdb: Samba4 IDMAP database
+    :param sid: user/group sid
+    :param xid: user/group id
+    :param xid_type: type of id (UID/GID)
+    :param logger: Logger object
+    """
 
     # First try to see if we already have this entry
     found = False
-    try:
-        msg = idmapdb.search(expression='objectSid=%s' % str(sid))
-        if msg.count == 1:
-            found = True
-    except Exception, e:
-        raise e
+    msg = idmapdb.search(expression='objectSid=%s' % str(sid))
+    if msg.count == 1:
+        found = True
 
     if found:
-        print msg.count
-        print dir(msg)
         try:
             m = ldb.Message()
-            m.dn = ldb.Dn(idmapdb, msg[0]['dn'])
+            m.dn = msg[0]['dn']
             m['xidNumber'] = ldb.MessageElement(str(xid), ldb.FLAG_MOD_REPLACE, 'xidNumber')
             m['type'] = ldb.MessageElement(xid_type, ldb.FLAG_MOD_REPLACE, 'type')
             idmapdb.modify(m)
         except ldb.LdbError, e:
             logger.warn('Could not modify idmap entry for sid=%s, id=%s, type=%s (%s)',
                             str(sid), str(xid), xid_type, str(e))
-        except Exception, e:
-            raise e
     else:
         try:
             idmapdb.add({"dn": "CN=%s" % str(sid),
@@ -187,16 +107,22 @@ def add_idmap_entry(idmapdb, sid, xid, xid_type, logger):
         except ldb.LdbError, e:
             logger.warn('Could not add idmap entry for sid=%s, id=%s, type=%s (%s)',
                             str(sid), str(xid), xid_type, str(e))
-        except Exception, e:
-            raise e
 
 
-def import_idmap(idmapdb, samba3_idmap, logger):
+def import_idmap(idmapdb, samba3, logger):
     """Import idmap data.
 
-    :param samba3_idmap: Samba 3 IDMAP database to import from
+    :param idmapdb: Samba4 IDMAP database
+    :param samba3_idmap: Samba3 IDMAP database to import from
+    :param logger: Logger object
     """
 
+    try:
+        samba3_idmap = samba3.get_idmap_db()
+    except IOError as (errno, strerror):
+        logger.warn('Cannot open idmap database, Ignoring: ({0}): {1}'.format(errno, strerror))
+        return
+
     currentxid = max(samba3_idmap.get_user_hwm(), samba3_idmap.get_group_hwm())
     lowerbound = currentxid
     # FIXME: upperbound
@@ -221,7 +147,12 @@ def import_idmap(idmapdb, samba3_idmap, logger):
 
 
 def add_group_from_mapping_entry(samdb, groupmap, logger):
-    """Add or modify group from group mapping entry"""
+    """Add or modify group from group mapping entry
+
+    param samdb: Samba4 SAM database
+    param groupmap: Groupmap entry
+    param logger: Logger object
+    """
 
     # First try to see if we already have this entry
     try:
@@ -232,15 +163,16 @@ def add_group_from_mapping_entry(samdb, groupmap, logger):
             found = False
         else:
             raise ldb.LdbError(ecode, emsg)
-    except Exception, e:
-        raise e
 
     if found:
         logger.warn('Group already exists sid=%s, groupname=%s existing_groupname=%s, Ignoring.',
                             str(groupmap.sid), groupmap.nt_name, msg[0]['sAMAccountName'][0])
     else:
         if groupmap.sid_name_use == lsa.SID_NAME_WKN_GRP:
-            return
+            # In a lot of Samba3 databases, aliases are marked as well known groups
+            (group_dom_sid, rid) = group.sid.split()
+            if (group_dom_sid != security.dom_sid(security.SID_BUILTIN)):
+                return
 
         m = ldb.Message()
         m.dn = ldb.Dn(samdb, "CN=%s,CN=Users,%s" % (groupmap.nt_name, samdb.get_default_basedn()))
@@ -250,31 +182,38 @@ def add_group_from_mapping_entry(samdb, groupmap, logger):
         m['a04'] = ldb.MessageElement(groupmap.comment, ldb.FLAG_MOD_ADD, 'description')
         m['a05'] = ldb.MessageElement(groupmap.nt_name, ldb.FLAG_MOD_ADD, 'sAMAccountName')
 
-        if groupmap.sid_name_use == lsa.SID_NAME_ALIAS:
+        # Fix up incorrect 'well known' groups that are actually builtin (per test above) to be aliases
+        if groupmap.sid_name_use == lsa.SID_NAME_ALIAS or groupmap.sid_name_use == lsa.SID_NAME_WKN_GRP:
             m['a06'] = ldb.MessageElement(str(dsdb.GTYPE_SECURITY_DOMAIN_LOCAL_GROUP), ldb.FLAG_MOD_ADD, 'groupType')
 
         try:
             samdb.add(m, controls=["relax:0"])
         except ldb.LdbError, e:
             logger.warn('Could not add group name=%s (%s)', groupmap.nt_name, str(e))
-        except Exception, e:
-            raise(e)
 
 
 def add_users_to_group(samdb, group, members, logger):
-    """Add user/member to group/alias"""
+    """Add user/member to group/alias
 
+    param samdb: Samba4 SAM database
+    param group: Groupmap object
+    param members: List of member SIDs
+    param logger: Logger object
+    """
     for member_sid in members:
         m = ldb.Message()
-        m.dn = ldb.Dn(samdb, "<SID=%s" % str(group.sid))
-        m['a01'] = ldb.MessageElement("<SID=%s>" % str(member_sid), ldb.FLAG_MOD_REPLACE, 'member')
+        m.dn = ldb.Dn(samdb, "<SID=%s>" % str(group.sid))
+        m['a01'] = ldb.MessageElement("<SID=%s>" % str(member_sid), ldb.FLAG_MOD_ADD, 'member')
 
         try:
             samdb.modify(m)
-        except ldb.LdbError, e:
-            logger.warn("Could not add member to group '%s'", groupmap.nt_name)
-        except Exception, e:
-            raise(e)
+        except ldb.LdbError, (ecode, emsg):
+            if ecode == ldb.ERR_ENTRY_ALREADY_EXISTS:
+                logger.info("skipped re-adding member '%s' to group '%s': %s", member_sid, group.sid, emsg)
+            elif ecode == ldb.ERR_NO_SUCH_OBJECT:
+                raise ProvisioningError("Could not add member '%s' to group '%s' as either group or user record doesn't exist: %s" % (member_sid, group.sid, emsg))
+            else:
+                raise ProvisioningError("Could not add member '%s' to group '%s': %s" % (member_sid, group.sid, emsg))
 
 
 def import_wins(samba4_winsdb, samba3_winsdb):
@@ -484,38 +423,38 @@ def import_registry(samba4_registry, samba3_regdb):
             key_handle.set_value(value_name, value_type, value_data)
 
 
-def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
+def upgrade_from_samba3(samba3, logger, targetdir, session_info=None, useeadb=False):
     """Upgrade from samba3 database to samba4 AD database
-    """
 
-    # Read samba3 smb.conf
-    oldconf = s3param.get_context();
-    oldconf.load(smbconf)
+    :param samba3: samba3 object
+    :param logger: Logger object
+    :param targetdir: samba4 database directory
+    :param session_info: Session information
+    """
 
-    if oldconf.get("domain logons"):
+    if samba3.lp.get("domain logons"):
         serverrole = "domain controller"
     else:
-        if oldconf.get("security") == "user":
+        if samba3.lp.get("security") == "user":
             serverrole = "standalone"
         else:
             serverrole = "member server"
 
-    domainname = oldconf.get("workgroup")
-    realm = oldconf.get("realm")
-    netbiosname = oldconf.get("netbios name")
+    domainname = samba3.lp.get("workgroup")
+    realm = samba3.lp.get("realm")
+    netbiosname = samba3.lp.get("netbios name")
 
     # secrets db
     secrets_db = samba3.get_secrets_db()
 
     if not domainname:
         domainname = secrets_db.domains()[0]
-        logger.warning("No domain specified in smb.conf file, assuming '%s'",
+        logger.warning("No workgroup specified in smb.conf file, assuming '%s'",
                 domainname)
 
     if not realm:
-        if oldconf.get("domain logons"):
-            logger.warning("No realm specified in smb.conf file and being a DC. That upgrade path doesn't work! Please add a 'realm' directive to your old smb.conf to let us know which one you want to use (generally it's the upcased DNS domainname).")
-            return
+        if serverrole == "domain controller":
+            raise ProvisioningError("No realm specified in smb.conf file and being a DC. That upgrade path doesn't work! Please add a 'realm' directive to your old smb.conf to let us know which one you want to use (it is the DNS name of the AD domain you wish to create.")
         else:
             realm = domainname.upper()
             logger.warning("No realm specified in smb.conf file, assuming '%s'",
@@ -535,27 +474,30 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
     # We must close the direct pytdb database before the C code loads it
     secrets_db.close()
 
-    passdb.set_secrets_dir(samba3.privatedir)
+    # Connect to old password backend
+    passdb.set_secrets_dir(samba3.lp.get("private dir"))
+    s3db = samba3.get_sam_db()
 
     # Get domain sid
     try:
         domainsid = passdb.get_global_sam_sid()
-    except:
+    except passdb.error:
         raise Exception("Can't find domain sid for '%s', Exiting." % domainname)
 
     # Get machine account, sid, rid
     try:
-        machineacct = old_passdb.getsampwnam('%s$' % netbiosname)
+        machineacct = s3db.getsampwnam('%s$' % netbiosname)
         machinesid, machinerid = machineacct.user_sid.split()
     except:
         pass
 
-    # Connect to old password backend
-    old_passdb = passdb.PDB(oldconf.get('passdb backend'))
+    # Export account policy
+    logger.info("Exporting account policy")
+    policy = s3db.get_account_policy()
 
-    # Import groups from old passdb backend
+    # Export groups from old passdb backend
     logger.info("Exporting groups")
-    grouplist = old_passdb.enum_group_mapping()
+    grouplist = s3db.enum_group_mapping()
     groupmembers = {}
     for group in grouplist:
         sid, rid = group.sid.split()
@@ -564,23 +506,31 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
                next_rid = rid + 1
 
         # Get members for each group/alias
-        if group.sid_name_use == lsa.SID_NAME_ALIAS or group.sid_name_use == lsa.SID_NAME_WKN_GRP:
-            members = old_passdb.enum_aliasmem(group.sid)
+        if group.sid_name_use == lsa.SID_NAME_ALIAS:
+            members = s3db.enum_aliasmem(group.sid)
         elif group.sid_name_use == lsa.SID_NAME_DOM_GRP:
             try:
-                members = old_passdb.enum_group_members(group.sid)
+                members = s3db.enum_group_members(group.sid)
             except:
                 continue
+            groupmembers[group.nt_name] = members
+        elif group.sid_name_use == lsa.SID_NAME_WKN_GRP:
+            (group_dom_sid, rid) = group.sid.split()
+            if (group_dom_sid != security.dom_sid(security.SID_BUILTIN)):
+                logger.warn("Ignoring 'well known' group '%s' (should already be in AD, and have no members)",
+                            group.nt_name)
+                continue
+            # A number of buggy databases mix up well known groups and aliases.
+            members = s3db.enum_aliasmem(group.sid)
         else:
             logger.warn("Ignoring group '%s' with sid_name_use=%d",
                         group.nt_name, group.sid_name_use)
             continue
-        groupmembers[group.nt_name] = members
 
 
-    # Import users from old passdb backend
+    # Export users from old passdb backend
     logger.info("Exporting users")
-    userlist = old_passdb.search_users(0)
+    userlist = s3db.search_users(0)
     userdata = {}
     uids = {}
     admin_user = None
@@ -593,10 +543,29 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
             continue
         if entry['rid'] >= next_rid:
             next_rid = entry['rid'] + 1
+
+        user = s3db.getsampwnam(username)
+        acct_type = (user.acct_ctrl & (samr.ACB_NORMAL|samr.ACB_WSTRUST|samr.ACB_SVRTRUST|samr.ACB_DOMTRUST))
+        if (acct_type == samr.ACB_NORMAL or acct_type == samr.ACB_WSTRUST or acct_type == samr.ACB_SVRTRUST):
+            pass
+        elif acct_type == samr.ACB_DOMTRUST:
+            logger.warn("  Skipping inter-domain trust from domain %s, this trust must be re-created as an AD trust" % username[:-1])
+            continue
+        elif acct_type == (samr.ACB_NORMAL|samr.ACB_WSTRUST) and username[-1] == '$':
+            logger.warn("  Fixing account %s which had both ACB_NORMAL (U) and ACB_WSTRUST (W) set.  Account will be marked as ACB_WSTRUST (W), i.e. as a domain member" % username)
+            user.acct_ctrl = (user.acct_ctrl & ~samr.ACB_NORMAL)
+        else:
+            raise ProvisioningError("""Failed to upgrade due to invalid account %s, account control flags 0x%08X must have exactly one of
+ACB_NORMAL (N, 0x%08X), ACB_WSTRUST (W 0x%08X), ACB_SVRTRUST (S 0x%08X) or ACB_DOMTRUST (D 0x%08X).
+
+Please fix this account before attempting to upgrade again
+"""
+                                    % (user.acct_flags, username,
+                                       samr.ACB_NORMAL, samr.ACB_WSTRUST, samr.ACB_SVRTRUST, samr.ACB_DOMTRUST))
         
-        userdata[username] = old_passdb.getsampwnam(username)
+        userdata[username] = user
         try:
-            uids[username] = old_passdb.sid_to_id(userdata[username].user_sid)[0]
+            uids[username] = s3db.sid_to_id(user.user_sid)[0]
         except:
             try:
                 uids[username] = pwd.getpwnam(username).pw_uid
@@ -608,7 +577,6 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
         if username.lower() == 'administrator':
             admin_user = username
 
-
     logger.info("Next rid = %d", next_rid)
 
     # Do full provision
@@ -617,21 +585,30 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
                        domainsid=str(domainsid), next_rid=next_rid,
                        dc_rid=machinerid,
                        hostname=netbiosname, machinepass=machinepass,
-                       serverrole=serverrole, samdb_fill=FILL_FULL)
+                       serverrole=serverrole, samdb_fill=FILL_FULL,
+                       useeadb=useeadb)
 
-    logger.info("Import WINS")
+    # Import WINS database
+    logger.info("Importing WINS database")
     import_wins(Ldb(result.paths.winsdb), samba3.get_wins_db())
 
-    new_smbconf = result.lp.configfile
-    newconf = s3param.get_context()
-    newconf.load(new_smbconf)
+    # Set Account policy
+    logger.info("Importing Account policy")
+    import_sam_policy(result.samdb, policy, logger)
+
+    # Migrate IDMAP database
+    logger.info("Importing idmap database")
+    import_idmap(result.idmap, samba3, logger)
 
-    # Migrate idmap
-    logger.info("Migrating idmap database")
-    import_idmap(result.idmap, samba3.get_idmap_db(), logger)
+    # Set the s3 context for samba4 configuration
+    new_lp_ctx = s3param.get_context()
+    new_lp_ctx.load(result.lp.configfile)
+    new_lp_ctx.set("private dir", result.lp.get("private dir"))
+    new_lp_ctx.set("state directory", result.lp.get("state directory"))
+    new_lp_ctx.set("lock directory", result.lp.get("lock directory"))
 
     # Connect to samba4 backend
-    new_passdb = passdb.PDB('samba4')
+    s4_passdb = passdb.PDB(new_lp_ctx.get("passdb backend"))
 
     # Export groups to samba4 backend
     logger.info("Importing groups")
@@ -646,7 +623,7 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
     for username in userdata:
         if username.lower() == 'administrator' or username.lower() == 'root':
             continue
-        new_passdb.add_sam_account(userdata[username])
+        s4_passdb.add_sam_account(userdata[username])
         if username in uids:
             add_idmap_entry(result.idmap, userdata[username].user_sid, uids[username], "UID", logger)
 
@@ -658,14 +635,15 @@ def upgrade_from_samba3(samba3, logger, session_info, smbconf, targetdir):
     # Set password for administrator
     if admin_user:
         logger.info("Setting password for administrator")
-        admin_userdata = new_passdb.getsampwnam("administrator")
+        admin_userdata = s4_passdb.getsampwnam("administrator")
         admin_userdata.nt_passwd = userdata[admin_user].nt_passwd
         if userdata[admin_user].lanman_passwd:
             admin_userdata.lanman_passwd = userdata[admin_user].lanman_passwd
         admin_userdata.pass_last_set_time = userdata[admin_user].pass_last_set_time
         if userdata[admin_user].pw_history:
             admin_userdata.pw_history = userdata[admin_user].pw_history
-        new_passdb.update_sam_account(admin_userdata)
+        s4_passdb.update_sam_account(admin_userdata)
         logger.info("Administrator password has been set to password of user '%s'", admin_user)
 
     # FIXME: import_registry(registry.Registry(), samba3.get_registry())
+    # FIXME: shares