netcmd/ldapcmp: use set instead of list to find missing DNs
[amitay/samba.git] / python / samba / netcmd / ldapcmp.py
1 # Unix SMB/CIFS implementation.
2 # A command to compare differences of objects and attributes between
3 # two LDAP servers both running at the same time. It generally compares
4 # one of the three pratitions DOMAIN, CONFIGURATION or SCHEMA. Users
5 # that have to be provided sheould be able to read objects in any of the
6 # above partitions.
7
8 # Copyright (C) Zahari Zahariev <zahari.zahariev@postpath.com> 2009, 2010
9 #
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation; either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the GNU General Public License
21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
22 #
23
24 import os
25 import re
26 import sys
27
28 import samba
29 import samba.getopt as options
30 from samba import Ldb
31 from samba.ndr import ndr_unpack
32 from samba.dcerpc import security
33 from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, ERR_NO_SUCH_OBJECT, LdbError
34 from samba.netcmd import (
35     Command,
36     CommandError,
37     Option,
38 )
39
40 global summary
41 summary = {}
42
43
44 class LDAPBase(object):
45
46     def __init__(self, host, creds, lp,
47                  two=False, quiet=False, descriptor=False, sort_aces=False, verbose=False,
48                  view="section", base="", scope="SUB",
49                  outf=sys.stdout, errf=sys.stderr, skip_missing_dn=True):
50         ldb_options = []
51         samdb_url = host
52         if "://" not in host:
53             if os.path.isfile(host):
54                 samdb_url = "tdb://%s" % host
55             else:
56                 samdb_url = "ldap://%s" % host
57         # use 'paged_search' module when connecting remotely
58         if samdb_url.lower().startswith("ldap://"):
59             ldb_options = ["modules:paged_searches"]
60         self.outf = outf
61         self.errf = errf
62         self.ldb = Ldb(url=samdb_url,
63                        credentials=creds,
64                        lp=lp,
65                        options=ldb_options)
66         self.search_base = base
67         self.search_scope = scope
68         self.two_domains = two
69         self.quiet = quiet
70         self.descriptor = descriptor
71         self.sort_aces = sort_aces
72         self.view = view
73         self.verbose = verbose
74         self.host = host
75         self.skip_missing_dn = skip_missing_dn
76         self.base_dn = str(self.ldb.get_default_basedn())
77         self.root_dn = str(self.ldb.get_root_basedn())
78         self.config_dn = str(self.ldb.get_config_basedn())
79         self.schema_dn = str(self.ldb.get_schema_basedn())
80         self.domain_netbios = self.find_netbios()
81         self.server_names = self.find_servers()
82         self.domain_name = re.sub("[Dd][Cc]=", "", self.base_dn).replace(",", ".")
83         self.domain_sid = self.find_domain_sid()
84         self.get_sid_map()
85         #
86         # Log some domain controller specific place-holers that are being used
87         # when compare content of two DCs. Uncomment for DEBUG purposes.
88         if self.two_domains and not self.quiet:
89             self.outf.write("\n* Place-holders for %s:\n" % self.host)
90             self.outf.write(4 * " " + "${DOMAIN_DN}      => %s\n" %
91                             self.base_dn)
92             self.outf.write(4 * " " + "${DOMAIN_NETBIOS} => %s\n" %
93                             self.domain_netbios)
94             self.outf.write(4 * " " + "${SERVER_NAME}     => %s\n" %
95                             self.server_names)
96             self.outf.write(4 * " " + "${DOMAIN_NAME}    => %s\n" %
97                             self.domain_name)
98
99     def find_domain_sid(self):
100         res = self.ldb.search(base=self.base_dn, expression="(objectClass=*)", scope=SCOPE_BASE)
101         return ndr_unpack(security.dom_sid, res[0]["objectSid"][0])
102
103     def find_servers(self):
104         """
105         """
106         res = self.ldb.search(base="OU=Domain Controllers,%s" % self.base_dn,
107                               scope=SCOPE_SUBTREE, expression="(objectClass=computer)", attrs=["cn"])
108         assert len(res) > 0
109         srv = []
110         for x in res:
111             srv.append(str(x["cn"][0]))
112         return srv
113
114     def find_netbios(self):
115         res = self.ldb.search(base="CN=Partitions,%s" % self.config_dn,
116                               scope=SCOPE_SUBTREE, attrs=["nETBIOSName"])
117         assert len(res) > 0
118         for x in res:
119             if "nETBIOSName" in x.keys():
120                 return x["nETBIOSName"][0]
121
122     def object_exists(self, object_dn):
123         res = None
124         try:
125             res = self.ldb.search(base=object_dn, scope=SCOPE_BASE)
126         except LdbError as e2:
127             (enum, estr) = e2.args
128             if enum == ERR_NO_SUCH_OBJECT:
129                 return False
130             raise
131         return len(res) == 1
132
133     def delete_force(self, object_dn):
134         try:
135             self.ldb.delete(object_dn)
136         except Ldb.LdbError as e:
137             assert "No such object" in str(e)
138
139     def get_attribute_name(self, key):
140         """ Returns the real attribute name
141             It resolved ranged results e.g. member;range=0-1499
142         """
143
144         r = re.compile("^([^;]+);range=(\d+)-(\d+|\*)$")
145
146         m = r.match(key)
147         if m is None:
148             return key
149
150         return m.group(1)
151
152     def get_attribute_values(self, object_dn, key, vals):
153         """ Returns list with all attribute values
154             It resolved ranged results e.g. member;range=0-1499
155         """
156
157         r = re.compile("^([^;]+);range=(\d+)-(\d+|\*)$")
158
159         m = r.match(key)
160         if m is None:
161             # no range, just return the values
162             return vals
163
164         attr = m.group(1)
165         hi = int(m.group(3))
166
167         # get additional values in a loop
168         # until we get a response with '*' at the end
169         while True:
170
171             n = "%s;range=%d-*" % (attr, hi + 1)
172             res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, attrs=[n])
173             assert len(res) == 1
174             res = dict(res[0])
175             del res["dn"]
176
177             fm = None
178             fvals = None
179
180             for key in res.keys():
181                 m = r.match(key)
182
183                 if m is None:
184                     continue
185
186                 if m.group(1) != attr:
187                     continue
188
189                 fm = m
190                 fvals = list(res[key])
191                 break
192
193             if fm is None:
194                 break
195
196             vals.extend(fvals)
197             if fm.group(3) == "*":
198                 # if we got "*" we're done
199                 break
200
201             assert int(fm.group(2)) == hi + 1
202             hi = int(fm.group(3))
203
204         return vals
205
206     def get_attributes(self, object_dn):
207         """ Returns dict with all default visible attributes
208         """
209         res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, attrs=["*"])
210         assert len(res) == 1
211         res = dict(res[0])
212         # 'Dn' element is not iterable and we have it as 'distinguishedName'
213         del res["dn"]
214         for key in list(res.keys()):
215             vals = list(res[key])
216             del res[key]
217             name = self.get_attribute_name(key)
218             res[name] = self.get_attribute_values(object_dn, key, vals)
219
220         return res
221
222     def get_descriptor_sddl(self, object_dn):
223         res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, attrs=["nTSecurityDescriptor"])
224         desc = res[0]["nTSecurityDescriptor"][0]
225         desc = ndr_unpack(security.descriptor, desc)
226         return desc.as_sddl(self.domain_sid)
227
228     def guid_as_string(self, guid_blob):
229         """ Translate binary representation of schemaIDGUID to standard string representation.
230             @gid_blob: binary schemaIDGUID
231         """
232         blob = "%s" % guid_blob
233         stops = [4, 2, 2, 2, 6]
234         index = 0
235         res = ""
236         x = 0
237         while x < len(stops):
238             tmp = ""
239             y = 0
240             while y < stops[x]:
241                 c = hex(ord(blob[index])).replace("0x", "")
242                 c = [None, "0" + c, c][len(c)]
243                 if 2 * index < len(blob):
244                     tmp = c + tmp
245                 else:
246                     tmp += c
247                 index += 1
248                 y += 1
249             res += tmp + " "
250             x += 1
251         assert index == len(blob)
252         return res.strip().replace(" ", "-")
253
254     def get_sid_map(self):
255         """ Build dictionary that maps GUID to 'name' attribute found in Schema or Extended-Rights.
256         """
257         self.sid_map = {}
258         res = self.ldb.search(base=self.base_dn,
259                               expression="(objectSid=*)", scope=SCOPE_SUBTREE, attrs=["objectSid", "sAMAccountName"])
260         for item in res:
261             try:
262                 self.sid_map["%s" % ndr_unpack(security.dom_sid, item["objectSid"][0])] = str(item["sAMAccountName"][0])
263             except KeyError:
264                 pass
265
266
267 class Descriptor(object):
268     def __init__(self, connection, dn, outf=sys.stdout, errf=sys.stderr):
269         self.outf = outf
270         self.errf = errf
271         self.con = connection
272         self.dn = dn
273         self.sddl = self.con.get_descriptor_sddl(self.dn)
274         self.dacl_list = self.extract_dacl()
275         if self.con.sort_aces:
276             self.dacl_list.sort()
277
278     def extract_dacl(self):
279         """ Extracts the DACL as a list of ACE string (with the brakets).
280         """
281         try:
282             if "S:" in self.sddl:
283                 res = re.search("D:(.*?)(\(.*?\))S:", self.sddl).group(2)
284             else:
285                 res = re.search("D:(.*?)(\(.*\))", self.sddl).group(2)
286         except AttributeError:
287             return []
288         return re.findall("(\(.*?\))", res)
289
290     def fix_sid(self, ace):
291         res = "%s" % ace
292         sids = re.findall("S-[-0-9]+", res)
293         # If there are not SIDs to replace return the same ACE
294         if len(sids) == 0:
295             return res
296         for sid in sids:
297             try:
298                 name = self.con.sid_map[sid]
299                 res = res.replace(sid, name)
300             except KeyError:
301                 # Do not bother if the SID is not found in baseDN
302                 pass
303         return res
304
305     def diff_1(self, other):
306         res = ""
307         if len(self.dacl_list) != len(other.dacl_list):
308             res += 4 * " " + "Difference in ACE count:\n"
309             res += 8 * " " + "=> %s\n" % len(self.dacl_list)
310             res += 8 * " " + "=> %s\n" % len(other.dacl_list)
311         #
312         i = 0
313         flag = True
314         while True:
315             self_ace = None
316             other_ace = None
317             try:
318                 self_ace = "%s" % self.dacl_list[i]
319             except IndexError:
320                 self_ace = ""
321             #
322             try:
323                 other_ace = "%s" % other.dacl_list[i]
324             except IndexError:
325                 other_ace = ""
326             if len(self_ace) + len(other_ace) == 0:
327                 break
328             self_ace_fixed = "%s" % self.fix_sid(self_ace)
329             other_ace_fixed = "%s" % other.fix_sid(other_ace)
330             if self_ace_fixed != other_ace_fixed:
331                 res += "%60s * %s\n" % (self_ace_fixed, other_ace_fixed)
332                 flag = False
333             else:
334                 res += "%60s | %s\n" % (self_ace_fixed, other_ace_fixed)
335             i += 1
336         return (flag, res)
337
338     def diff_2(self, other):
339         res = ""
340         if len(self.dacl_list) != len(other.dacl_list):
341             res += 4 * " " + "Difference in ACE count:\n"
342             res += 8 * " " + "=> %s\n" % len(self.dacl_list)
343             res += 8 * " " + "=> %s\n" % len(other.dacl_list)
344         #
345         common_aces = []
346         self_aces = []
347         other_aces = []
348         self_dacl_list_fixed = []
349         other_dacl_list_fixed = []
350         [self_dacl_list_fixed.append(self.fix_sid(ace)) for ace in self.dacl_list]
351         [other_dacl_list_fixed.append(other.fix_sid(ace)) for ace in other.dacl_list]
352         for ace in self_dacl_list_fixed:
353             try:
354                 other_dacl_list_fixed.index(ace)
355             except ValueError:
356                 self_aces.append(ace)
357             else:
358                 common_aces.append(ace)
359         self_aces = sorted(self_aces)
360         if len(self_aces) > 0:
361             res += 4 * " " + "ACEs found only in %s:\n" % self.con.host
362             for ace in self_aces:
363                 res += 8 * " " + ace + "\n"
364         #
365         for ace in other_dacl_list_fixed:
366             try:
367                 self_dacl_list_fixed.index(ace)
368             except ValueError:
369                 other_aces.append(ace)
370             else:
371                 common_aces.append(ace)
372         other_aces = sorted(other_aces)
373         if len(other_aces) > 0:
374             res += 4 * " " + "ACEs found only in %s:\n" % other.con.host
375             for ace in other_aces:
376                 res += 8 * " " + ace + "\n"
377         #
378         common_aces = sorted(list(set(common_aces)))
379         if self.con.verbose:
380             res += 4 * " " + "ACEs found in both:\n"
381             for ace in common_aces:
382                 res += 8 * " " + ace + "\n"
383         return (self_aces == [] and other_aces == [], res)
384
385
386 class LDAPObject(object):
387     def __init__(self, connection, dn, summary, filter_list,
388                  outf=sys.stdout, errf=sys.stderr):
389         self.outf = outf
390         self.errf = errf
391         self.con = connection
392         self.two_domains = self.con.two_domains
393         self.quiet = self.con.quiet
394         self.verbose = self.con.verbose
395         self.summary = summary
396         self.dn = dn.replace("${DOMAIN_DN}", self.con.base_dn)
397         self.dn = self.dn.replace("CN=${DOMAIN_NETBIOS}", "CN=%s" % self.con.domain_netbios)
398         for x in self.con.server_names:
399             self.dn = self.dn.replace("CN=${SERVER_NAME}", "CN=%s" % x)
400         self.attributes = self.con.get_attributes(self.dn)
401         # One domain - two domain controllers
402         #
403         # Some attributes are defined as FLAG_ATTR_NOT_REPLICATED
404         #
405         # The following list was generated by
406         # egrep '^systemFlags: |^ldapDisplayName: |^linkID: ' \
407         #       source4/setup/ad-schema/MS-AD_Schema_2K8_R2_Attributes.txt | \
408         #       grep -B1 FLAG_ATTR_NOT_REPLICATED | \
409         #       grep ldapDisplayName | \
410         #       cut -d ' ' -f2
411         self.non_replicated_attributes = [
412                 "badPasswordTime",
413                 "badPwdCount",
414                 "dSCorePropagationData",
415                 "lastLogoff",
416                 "lastLogon",
417                 "logonCount",
418                 "modifiedCount",
419                 "msDS-Cached-Membership",
420                 "msDS-Cached-Membership-Time-Stamp",
421                 "msDS-EnabledFeatureBL",
422                 "msDS-ExecuteScriptPassword",
423                 "msDS-NcType",
424                 "msDS-ReplicationEpoch",
425                 "msDS-RetiredReplNCSignatures",
426                 "msDS-USNLastSyncSuccess",
427                 # "distinguishedName", # This is implicitly replicated
428                 # "objectGUID", # This is implicitly replicated
429                 "partialAttributeDeletionList",
430                 "partialAttributeSet",
431                 "pekList",
432                 "prefixMap",
433                 "replPropertyMetaData",
434                 "replUpToDateVector",
435                 "repsFrom",
436                 "repsTo",
437                 "rIDNextRID",
438                 "rIDPreviousAllocationPool",
439                 "schemaUpdate",
440                 "serverState",
441                 "subRefs",
442                 "uSNChanged",
443                 "uSNCreated",
444                 "uSNLastObjRem",
445                 "whenChanged",  # This is implicitly replicated, but may diverge on updates of non-replicated attributes
446         ]
447         self.ignore_attributes = self.non_replicated_attributes
448         self.ignore_attributes += ["msExchServer1HighestUSN"]
449         if filter_list:
450             self.ignore_attributes += filter_list
451
452         self.dn_attributes = []
453         self.domain_attributes = []
454         self.servername_attributes = []
455         self.netbios_attributes = []
456         self.other_attributes = []
457         # Two domains - two domain controllers
458
459         if self.two_domains:
460             self.ignore_attributes += [
461                 "objectCategory", "objectGUID", "objectSid", "whenCreated",
462                 "whenChanged", "pwdLastSet", "uSNCreated", "creationTime",
463                 "modifiedCount", "priorSetTime", "rIDManagerReference",
464                 "gPLink", "ipsecNFAReference", "fRSPrimaryMember",
465                 "fSMORoleOwner", "masteredBy", "ipsecOwnersReference",
466                 "wellKnownObjects", "otherWellKnownObjects", "badPwdCount",
467                 "ipsecISAKMPReference", "ipsecFilterReference",
468                 "msDs-masteredBy", "lastSetTime",
469                 "ipsecNegotiationPolicyReference", "subRefs", "gPCFileSysPath",
470                 "accountExpires", "invocationId", "operatingSystemVersion",
471                 "oEMInformation",
472                 # After Exchange preps
473                 "targetAddress", "msExchMailboxGuid", "siteFolderGUID"]
474             #
475             # Attributes that contain the unique DN tail part e.g. 'DC=samba,DC=org'
476             self.dn_attributes = [
477                 "distinguishedName", "defaultObjectCategory", "member", "memberOf", "siteList", "nCName",
478                 "homeMDB", "homeMTA", "interSiteTopologyGenerator", "serverReference",
479                 "msDS-HasInstantiatedNCs", "hasMasterNCs", "msDS-hasMasterNCs", "msDS-HasDomainNCs", "dMDLocation",
480                 "msDS-IsDomainFor", "rIDSetReferences", "serverReferenceBL",
481                 # After Exchange preps
482                 "msExchHomeRoutingGroup", "msExchResponsibleMTAServer", "siteFolderServer", "msExchRoutingMasterDN",
483                 "msExchRoutingGroupMembersBL", "homeMDBBL", "msExchHomePublicMDB", "msExchOwningServer", "templateRoots",
484                 "addressBookRoots", "msExchPolicyRoots", "globalAddressList", "msExchOwningPFTree",
485                 "msExchResponsibleMTAServerBL", "msExchOwningPFTreeBL",
486                 # After 2012 R2 functional preparation
487                 "msDS-MembersOfResourcePropertyListBL",
488                 "msDS-ValueTypeReference",
489                 "msDS-MembersOfResourcePropertyList",
490                 "msDS-ValueTypeReferenceBL",
491                 "msDS-ClaimTypeAppliesToClass",
492             ]
493             self.dn_attributes = [x.upper() for x in self.dn_attributes]
494             #
495             # Attributes that contain the Domain name e.g. 'samba.org'
496             self.domain_attributes = [
497                 "proxyAddresses", "mail", "userPrincipalName", "msExchSmtpFullyQualifiedDomainName",
498                 "dnsHostName", "networkAddress", "dnsRoot", "servicePrincipalName", ]
499             self.domain_attributes = [x.upper() for x in self.domain_attributes]
500             #
501             # May contain DOMAIN_NETBIOS and SERVER_NAME
502             self.servername_attributes = ["distinguishedName", "name", "CN", "sAMAccountName", "dNSHostName",
503                                           "servicePrincipalName", "rIDSetReferences", "serverReference", "serverReferenceBL",
504                                           "msDS-IsDomainFor", "interSiteTopologyGenerator", ]
505             self.servername_attributes = [x.upper() for x in self.servername_attributes]
506             #
507             self.netbios_attributes = ["servicePrincipalName", "CN", "distinguishedName", "nETBIOSName", "name", ]
508             self.netbios_attributes = [x.upper() for x in self.netbios_attributes]
509             #
510             self.other_attributes = ["name", "DC", ]
511             self.other_attributes = [x.upper() for x in self.other_attributes]
512         #
513         self.ignore_attributes = [x.upper() for x in self.ignore_attributes]
514
515     def log(self, msg):
516         """
517         Log on the screen if there is no --quiet option set
518         """
519         if not self.quiet:
520             self.outf.write(msg +"\n")
521
522     def fix_dn(self, s):
523         res = "%s" % s
524         if not self.two_domains:
525             return res
526         if res.upper().endswith(self.con.base_dn.upper()):
527             res = res[:len(res) - len(self.con.base_dn)] + "${DOMAIN_DN}"
528         return res
529
530     def fix_domain_name(self, s):
531         res = "%s" % s
532         if not self.two_domains:
533             return res
534         res = res.replace(self.con.domain_name.lower(), self.con.domain_name.upper())
535         res = res.replace(self.con.domain_name.upper(), "${DOMAIN_NAME}")
536         return res
537
538     def fix_domain_netbios(self, s):
539         res = "%s" % s
540         if not self.two_domains:
541             return res
542         res = res.replace(self.con.domain_netbios.lower(), self.con.domain_netbios.upper())
543         res = res.replace(self.con.domain_netbios.upper(), "${DOMAIN_NETBIOS}")
544         return res
545
546     def fix_server_name(self, s):
547         res = "%s" % s
548         if not self.two_domains or len(self.con.server_names) > 1:
549             return res
550         for x in self.con.server_names:
551             res = res.upper().replace(x, "${SERVER_NAME}")
552         return res
553
554     def __eq__(self, other):
555         if self.con.descriptor:
556             return self.cmp_desc(other)
557         return self.cmp_attrs(other)
558
559     def cmp_desc(self, other):
560         d1 = Descriptor(self.con, self.dn, outf=self.outf, errf=self.errf)
561         d2 = Descriptor(other.con, other.dn, outf=self.outf, errf=self.errf)
562         if self.con.view == "section":
563             res = d1.diff_2(d2)
564         elif self.con.view == "collision":
565             res = d1.diff_1(d2)
566         else:
567             raise Exception("Unknown --view option value.")
568         #
569         self.screen_output = res[1][:-1]
570         other.screen_output = res[1][:-1]
571         #
572         return res[0]
573
574     def cmp_attrs(self, other):
575         res = ""
576         self.unique_attrs = []
577         self.df_value_attrs = []
578         other.unique_attrs = []
579         if self.attributes.keys() != other.attributes.keys():
580             #
581             title = 4 * " " + "Attributes found only in %s:" % self.con.host
582             for x in self.attributes.keys():
583                 if x not in other.attributes.keys() and \
584                         not x.upper() in [q.upper() for q in other.ignore_attributes]:
585                     if title:
586                         res += title + "\n"
587                         title = None
588                     res += 8 * " " + x + "\n"
589                     self.unique_attrs.append(x)
590             #
591             title = 4 * " " + "Attributes found only in %s:" % other.con.host
592             for x in other.attributes.keys():
593                 if x not in self.attributes.keys() and \
594                         not x.upper() in [q.upper() for q in self.ignore_attributes]:
595                     if title:
596                         res += title + "\n"
597                         title = None
598                     res += 8 * " " + x + "\n"
599                     other.unique_attrs.append(x)
600         #
601         missing_attrs = [x.upper() for x in self.unique_attrs]
602         missing_attrs += [x.upper() for x in other.unique_attrs]
603         title = 4 * " " + "Difference in attribute values:"
604         for x in self.attributes.keys():
605             if x.upper() in self.ignore_attributes or x.upper() in missing_attrs:
606                 continue
607             if isinstance(self.attributes[x], list) and isinstance(other.attributes[x], list):
608                 self.attributes[x] = sorted(self.attributes[x])
609                 other.attributes[x] = sorted(other.attributes[x])
610             if self.attributes[x] != other.attributes[x]:
611                 p = None
612                 q = None
613                 m = None
614                 n = None
615                 # First check if the difference can be fixed but shunting the first part
616                 # of the DomainHostName e.g. 'mysamba4.test.local' => 'mysamba4'
617                 if x.upper() in self.other_attributes:
618                     p = [self.con.domain_name.split(".")[0] == j for j in self.attributes[x]]
619                     q = [other.con.domain_name.split(".")[0] == j for j in other.attributes[x]]
620                     if p == q:
621                         continue
622                 # Attribute values that are list that contain DN based values that may differ
623                 elif x.upper() in self.dn_attributes:
624                     m = p
625                     n = q
626                     if not p and not q:
627                         m = self.attributes[x]
628                         n = other.attributes[x]
629                     p = [self.fix_dn(j) for j in m]
630                     q = [other.fix_dn(j) for j in n]
631                     if p == q:
632                         continue
633                 # Attributes that contain the Domain name in them
634                 if x.upper() in self.domain_attributes:
635                     m = p
636                     n = q
637                     if not p and not q:
638                         m = self.attributes[x]
639                         n = other.attributes[x]
640                     p = [self.fix_domain_name(j) for j in m]
641                     q = [other.fix_domain_name(j) for j in n]
642                     if p == q:
643                         continue
644                 #
645                 if x.upper() in self.servername_attributes:
646                     # Attributes with SERVER_NAME
647                     m = p
648                     n = q
649                     if not p and not q:
650                         m = self.attributes[x]
651                         n = other.attributes[x]
652                     p = [self.fix_server_name(j) for j in m]
653                     q = [other.fix_server_name(j) for j in n]
654                     if p == q:
655                         continue
656                 #
657                 if x.upper() in self.netbios_attributes:
658                     # Attributes with NETBIOS Domain name
659                     m = p
660                     n = q
661                     if not p and not q:
662                         m = self.attributes[x]
663                         n = other.attributes[x]
664                     p = [self.fix_domain_netbios(j) for j in m]
665                     q = [other.fix_domain_netbios(j) for j in n]
666                     if p == q:
667                         continue
668                 #
669                 if title:
670                     res += title + "\n"
671                     title = None
672                 if p and q:
673                     res += 8 * " " + x + " => \n%s\n%s" % (p, q) + "\n"
674                 else:
675                     res += 8 * " " + x + " => \n%s\n%s" % (self.attributes[x], other.attributes[x]) + "\n"
676                 self.df_value_attrs.append(x)
677         #
678         if self.unique_attrs + other.unique_attrs != []:
679             assert self.unique_attrs != other.unique_attrs
680         self.summary["unique_attrs"] += self.unique_attrs
681         self.summary["df_value_attrs"] += self.df_value_attrs
682         other.summary["unique_attrs"] += other.unique_attrs
683         other.summary["df_value_attrs"] += self.df_value_attrs  # they are the same
684         #
685         self.screen_output = res[:-1]
686         other.screen_output = res[:-1]
687         #
688         return res == ""
689
690
691 class LDAPBundle(object):
692
693     def __init__(self, connection, context, dn_list=None, filter_list=None,
694                  outf=sys.stdout, errf=sys.stderr):
695         self.outf = outf
696         self.errf = errf
697         self.con = connection
698         self.two_domains = self.con.two_domains
699         self.quiet = self.con.quiet
700         self.verbose = self.con.verbose
701         self.search_base = self.con.search_base
702         self.search_scope = self.con.search_scope
703         self.skip_missing_dn = self.con.skip_missing_dn
704         self.summary = {}
705         self.summary["unique_attrs"] = []
706         self.summary["df_value_attrs"] = []
707         self.summary["known_ignored_dn"] = []
708         self.summary["abnormal_ignored_dn"] = []
709         self.filter_list = filter_list
710         if dn_list:
711             self.dn_list = dn_list
712         elif context.upper() in ["DOMAIN", "CONFIGURATION", "SCHEMA", "DNSDOMAIN", "DNSFOREST"]:
713             self.context = context.upper()
714             self.dn_list = self.get_dn_list(context)
715         else:
716             raise Exception("Unknown initialization data for LDAPBundle().")
717         counter = 0
718         while counter < len(self.dn_list) and self.two_domains:
719             # Use alias reference
720             tmp = self.dn_list[counter]
721             tmp = tmp[:len(tmp) - len(self.con.base_dn)] + "${DOMAIN_DN}"
722             tmp = tmp.replace("CN=%s" % self.con.domain_netbios, "CN=${DOMAIN_NETBIOS}")
723             if len(self.con.server_names) == 1:
724                 for x in self.con.server_names:
725                     tmp = tmp.replace("CN=%s" % x, "CN=${SERVER_NAME}")
726             self.dn_list[counter] = tmp
727             counter += 1
728         self.dn_list = list(set(self.dn_list))
729         self.dn_list = sorted(self.dn_list)
730         self.size = len(self.dn_list)
731
732     def log(self, msg):
733         """
734         Log on the screen if there is no --quiet option set
735         """
736         if not self.quiet:
737             self.outf.write(msg + "\n")
738
739     def update_size(self):
740         self.size = len(self.dn_list)
741         self.dn_list = sorted(self.dn_list)
742
743     def diff(self, other):
744         res = True
745         if self.size != other.size:
746             self.log("\n* DN lists have different size: %s != %s" % (self.size, other.size))
747             if not self.skip_missing_dn:
748                 res = False
749
750         self_dns = set([q.upper() for q in self.dn_list])
751         other_dns = set([q.upper() for q in other.dn_list])
752
753         #
754         # This is the case where we want to explicitly compare two objects with different DNs.
755         # It does not matter if they are in the same DC, in two DC in one domain or in two
756         # different domains.
757         if self.search_scope != SCOPE_BASE and not self.skip_missing_dn:
758
759             self_only = self_dns - other_dns  # missing in other
760             if self_only:
761                 res = False
762                 self.log("\n* DNs found only in %s:" % self.con.host)
763                 for x in self_only:
764                     self.log(4 * " " + x)
765
766             other_only = other_dns - self_dns  # missing in self
767             if other_only:
768                 res = False
769                 self.log("\n* DNs found only in %s:" % other.con.host)
770                 for x in other_only:
771                     self.log(4 * " " + x)
772
773         common_dns = self_dns & other_dns
774         self.log("\n* Objects to be compared: %d" % len(common_dns))
775
776         for dn in common_dns:
777
778             try:
779                 object1 = LDAPObject(connection=self.con,
780                                      dn=dn,
781                                      summary=self.summary,
782                                      filter_list=self.filter_list,
783                                      outf=self.outf, errf=self.errf)
784             except LdbError as e:
785                 self.log("LdbError for dn %s: %s" % (dn, e))
786                 continue
787
788             try:
789                 object2 = LDAPObject(connection=other.con,
790                                      dn=dn,
791                                      summary=other.summary,
792                                      filter_list=self.filter_list,
793                                      outf=self.outf, errf=self.errf)
794             except LdbError as e:
795                 self.log("LdbError for dn %s: %s" % (dn, e))
796                 continue
797
798             if object1 == object2:
799                 if self.con.verbose:
800                     self.log("\nComparing:")
801                     self.log("'%s' [%s]" % (object1.dn, object1.con.host))
802                     self.log("'%s' [%s]" % (object2.dn, object2.con.host))
803                     self.log(4 * " " + "OK")
804             else:
805                 self.log("\nComparing:")
806                 self.log("'%s' [%s]" % (object1.dn, object1.con.host))
807                 self.log("'%s' [%s]" % (object2.dn, object2.con.host))
808                 self.log(object1.screen_output)
809                 self.log(4 * " " + "FAILED")
810                 res = False
811             self.summary = object1.summary
812             other.summary = object2.summary
813
814         return res
815
816     def get_dn_list(self, context):
817         """ Query LDAP server about the DNs of certain naming self.con.ext Domain (or Default), Configuration, Schema.
818             Parse all DNs and filter those that are 'strange' or abnormal.
819         """
820         if context.upper() == "DOMAIN":
821             search_base = self.con.base_dn
822         elif context.upper() == "CONFIGURATION":
823             search_base = self.con.config_dn
824         elif context.upper() == "SCHEMA":
825             search_base = self.con.schema_dn
826         elif context.upper() == "DNSDOMAIN":
827             search_base = "DC=DomainDnsZones,%s" % self.con.base_dn
828         elif context.upper() == "DNSFOREST":
829             search_base = "DC=ForestDnsZones,%s" % self.con.root_dn
830
831         dn_list = []
832         if not self.search_base:
833             self.search_base = search_base
834         self.search_scope = self.search_scope.upper()
835         if self.search_scope == "SUB":
836             self.search_scope = SCOPE_SUBTREE
837         elif self.search_scope == "BASE":
838             self.search_scope = SCOPE_BASE
839         elif self.search_scope == "ONE":
840             self.search_scope = SCOPE_ONELEVEL
841         else:
842             raise ValueError("Wrong 'scope' given. Choose from: SUB, ONE, BASE")
843         try:
844             res = self.con.ldb.search(base=self.search_base, scope=self.search_scope, attrs=["dn"])
845         except LdbError as e3:
846             (enum, estr) = e3.args
847             self.outf.write("Failed search of base=%s\n" % self.search_base)
848             raise
849         for x in res:
850             dn_list.append(x["dn"].get_linearized())
851         #
852         global summary
853         #
854         return dn_list
855
856     def print_summary(self):
857         self.summary["unique_attrs"] = list(set(self.summary["unique_attrs"]))
858         self.summary["df_value_attrs"] = list(set(self.summary["df_value_attrs"]))
859         #
860         if self.summary["unique_attrs"]:
861             self.log("\nAttributes found only in %s:" % self.con.host)
862             self.log("".join([str("\n" + 4 * " " + x) for x in self.summary["unique_attrs"]]))
863         #
864         if self.summary["df_value_attrs"]:
865             self.log("\nAttributes with different values:")
866             self.log("".join([str("\n" + 4 * " " + x) for x in self.summary["df_value_attrs"]]))
867             self.summary["df_value_attrs"] = []
868
869
870 class cmd_ldapcmp(Command):
871     """Compare two ldap databases."""
872     synopsis = "%prog <URL1> <URL2> (domain|configuration|schema|dnsdomain|dnsforest) [options]"
873
874     takes_optiongroups = {
875         "sambaopts": options.SambaOptions,
876         "versionopts": options.VersionOptions,
877         "credopts": options.CredentialsOptionsDouble,
878     }
879
880     takes_args = ["URL1", "URL2", "context1?", "context2?", "context3?", "context4?", "context5?"]
881
882     takes_options = [
883         Option("-w", "--two", dest="two", action="store_true", default=False,
884                help="Hosts are in two different domains"),
885         Option("-q", "--quiet", dest="quiet", action="store_true", default=False,
886                help="Do not print anything but relay on just exit code"),
887         Option("-v", "--verbose", dest="verbose", action="store_true", default=False,
888                help="Print all DN pairs that have been compared"),
889         Option("--sd", dest="descriptor", action="store_true", default=False,
890                help="Compare nTSecurityDescriptor attibutes only"),
891         Option("--sort-aces", dest="sort_aces", action="store_true", default=False,
892                help="Sort ACEs before comparison of nTSecurityDescriptor attribute"),
893         Option("--view", dest="view", default="section", choices=["section", "collision"],
894                help="Display mode for nTSecurityDescriptor results. Possible values: section or collision."),
895         Option("--base", dest="base", default="",
896                help="Pass search base that will build DN list for the first DC."),
897         Option("--base2", dest="base2", default="",
898                help="Pass search base that will build DN list for the second DC. Used when --two or when compare two different DNs."),
899         Option("--scope", dest="scope", default="SUB", choices=["SUB", "ONE", "BASE"],
900                help="Pass search scope that builds DN list. Options: SUB, ONE, BASE"),
901         Option("--filter", dest="filter", default="",
902                help="List of comma separated attributes to ignore in the comparision"),
903         Option("--skip-missing-dn", dest="skip_missing_dn", action="store_true", default=False,
904                help="Skip report and failure due to missing DNs in one server or another"),
905     ]
906
907     def run(self, URL1, URL2,
908             context1=None, context2=None, context3=None, context4=None, context5=None,
909             two=False, quiet=False, verbose=False, descriptor=False, sort_aces=False,
910             view="section", base="", base2="", scope="SUB", filter="",
911             credopts=None, sambaopts=None, versionopts=None, skip_missing_dn=False):
912
913         lp = sambaopts.get_loadparm()
914
915         using_ldap = URL1.startswith("ldap") or URL2.startswith("ldap")
916
917         if using_ldap:
918             creds = credopts.get_credentials(lp, fallback_machine=True)
919         else:
920             creds = None
921         creds2 = credopts.get_credentials2(lp, guess=False)
922         if creds2.is_anonymous():
923             creds2 = creds
924         else:
925             creds2.set_domain("")
926             creds2.set_workstation("")
927         if using_ldap and not creds.authentication_requested():
928             raise CommandError("You must supply at least one username/password pair")
929
930         # make a list of contexts to compare in
931         contexts = []
932         if context1 is None:
933             if base and base2:
934                 # If search bases are specified context is defaulted to
935                 # DOMAIN so the given search bases can be verified.
936                 contexts = ["DOMAIN"]
937             else:
938                 # if no argument given, we compare all contexts
939                 contexts = ["DOMAIN", "CONFIGURATION", "SCHEMA", "DNSDOMAIN", "DNSFOREST"]
940         else:
941             for c in [context1, context2, context3, context4, context5]:
942                 if c is None:
943                     continue
944                 if not c.upper() in ["DOMAIN", "CONFIGURATION", "SCHEMA", "DNSDOMAIN", "DNSFOREST"]:
945                     raise CommandError("Incorrect argument: %s" % c)
946                 contexts.append(c.upper())
947
948         if verbose and quiet:
949             raise CommandError("You cannot set --verbose and --quiet together")
950         if (not base and base2) or (base and not base2):
951             raise CommandError("You need to specify both --base and --base2 at the same time")
952
953         con1 = LDAPBase(URL1, creds, lp,
954                         two=two, quiet=quiet, descriptor=descriptor, sort_aces=sort_aces,
955                         verbose=verbose, view=view, base=base, scope=scope,
956                         outf=self.outf, errf=self.errf)
957         assert len(con1.base_dn) > 0
958
959         con2 = LDAPBase(URL2, creds2, lp,
960                         two=two, quiet=quiet, descriptor=descriptor, sort_aces=sort_aces,
961                         verbose=verbose, view=view, base=base2, scope=scope,
962                         outf=self.outf, errf=self.errf)
963         assert len(con2.base_dn) > 0
964
965         filter_list = filter.split(",")
966
967         status = 0
968         for context in contexts:
969             if not quiet:
970                 self.outf.write("\n* Comparing [%s] context...\n" % context)
971
972             b1 = LDAPBundle(con1, context=context, filter_list=filter_list,
973                             outf=self.outf, errf=self.errf)
974             b2 = LDAPBundle(con2, context=context, filter_list=filter_list,
975                             outf=self.outf, errf=self.errf)
976
977             if b1.diff(b2):
978                 if not quiet:
979                     self.outf.write("\n* Result for [%s]: SUCCESS\n" %
980                                     context)
981             else:
982                 if not quiet:
983                     self.outf.write("\n* Result for [%s]: FAILURE\n" % context)
984                     if not descriptor:
985                         assert len(b1.summary["df_value_attrs"]) == len(b2.summary["df_value_attrs"])
986                         b2.summary["df_value_attrs"] = []
987                         self.outf.write("\nSUMMARY\n")
988                         self.outf.write("---------\n")
989                         b1.print_summary()
990                         b2.print_summary()
991                 # mark exit status as FAILURE if a least one comparison failed
992                 status = -1
993         if status != 0:
994             raise CommandError("Compare failed: %d" % status)