Merge branch 'v4-0-stable' into newmaster
[kai/samba.git] / source4 / scripting / devel / ldapcmp
1 #!/usr/bin/env python
2 #
3 # Unix SMB/CIFS implementation.
4 # A script to compare differences of objects and attributes between
5 # two LDAP servers both running at the same time. It generally compares
6 # one of the three pratitions DOMAIN, CONFIGURATION or SCHEMA. Users
7 # that have to be provided sheould be able to read objects in any of the
8 # above partitions.
9
10 # Copyright (C) Zahari Zahariev <zahari.zahariev@postpath.com> 2009, 2010
11 #
12 # This program is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 3 of the License, or
15 # (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
24 #
25
26 import os
27 import re
28 import sys
29 from optparse import OptionParser
30
31 sys.path.insert(0, "bin/python")
32
33 import samba
34 import samba.getopt as options
35 from samba import Ldb
36 from samba.ndr import ndr_pack, ndr_unpack
37 from samba.dcerpc import security
38 from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, ERR_NO_SUCH_OBJECT, LdbError
39
40 global summary
41 summary = {}
42
43 class LDAPBase(object):
44
45     def __init__(self, host, cmd_opts, creds, lp):
46         if not "://" in host:
47             self.host = "ldap://" + host + ":389"
48         self.ldb = Ldb(self.host, credentials=creds, lp=lp,
49                              options=["modules:paged_searches"])
50         self.two_domains = cmd_opts.two
51         self.quiet = cmd_opts.quiet
52         self.host = host
53         self.base_dn = self.find_basedn()
54         self.domain_netbios = self.find_netbios()
55         self.server_names = self.find_servers()
56         self.domain_name = re.sub("[Dd][Cc]=", "", self.base_dn).replace(",", ".")
57         self.domain_sid_bin = self.get_object_sid(self.base_dn)
58         #
59         # Log some domain controller specific place-holers that are being used
60         # when compare content of two DCs. Uncomment for DEBUG purposes.
61         if self.two_domains and not self.quiet:
62             print "\n* Place-holders for %s:" % self.host
63             print 4*" " + "${DOMAIN_DN}      => %s" % self.base_dn
64             print 4*" " + "${DOMAIN_NETBIOS} => %s" % self.domain_netbios
65             print 4*" " + "${SERVERNAME}     => %s" % self.server_names
66             print 4*" " + "${DOMAIN_NAME}    => %s" % self.domain_name
67
68     def find_servers(self):
69         """
70         """
71         res = self.ldb.search(base="OU=Domain Controllers,%s" % self.base_dn, \
72                 scope=SCOPE_SUBTREE, expression="(objectClass=computer)", attrs=["cn"])
73         assert len(res) > 0
74         srv = []
75         for x in res:
76             srv.append(x["cn"][0])
77         return srv
78
79     def find_netbios(self):
80         res = self.ldb.search(base="CN=Partitions,CN=Configuration,%s" % self.base_dn, \
81                 scope=SCOPE_SUBTREE, attrs=["nETBIOSName"])
82         assert len(res) > 0
83         for x in res:
84             if "nETBIOSName" in x.keys():
85                 return x["nETBIOSName"][0]
86
87     def find_basedn(self):
88         res = self.ldb.search(base="", expression="(objectClass=*)", scope=SCOPE_BASE,
89                 attrs=["defaultNamingContext"])
90         assert len(res) == 1
91         return res[0]["defaultNamingContext"][0]
92
93     def object_exists(self, object_dn):
94         res = None
95         try:
96             res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, expression="(objectClass=*)")
97         except LdbError, (ERR_NO_SUCH_OBJECT, _):
98             return False
99         return len(res) == 1
100
101     def get_object_sid(self, object_dn):
102         try:
103             res = self.ldb.search(base=object_dn, expression="(objectClass=*)", scope=SCOPE_BASE, attrs=["objectSid"])
104         except LdbError, (ERR_NO_SUCH_OBJECT, _):
105             raise Exception("DN sintax is wrong or object does't exist: " + object_dn)
106         assert len(res) == 1
107         return res[0]["objectSid"][0]
108
109     def delete_force(self, object_dn):
110         try:
111             self.ldb.delete(object_dn)
112         except Ldb.LdbError, e:
113             assert "No such object" in str(e)
114
115     def get_attributes(self, object_dn):
116         """ Returns dict with all default visible attributes
117         """
118         res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, attrs=["*"])
119         assert len(res) == 1
120         res = dict(res[0])
121         # 'Dn' element is not iterable and we have it as 'distinguishedName'
122         del res["dn"]
123         for key in res.keys():
124             res[key] = list(res[key])
125         return res
126
127     def get_descriptor(self, object_dn):
128         res = self.ldb.search(base=object_dn, scope=SCOPE_BASE, attrs=["nTSecurityDescriptor"])
129         return res[0]["nTSecurityDescriptor"][0]
130
131
132 class LDAPObject(object):
133     def __init__(self, connection, dn, summary, cmd_opts):
134         self.con = connection
135         self.two_domains = cmd_opts.two
136         self.quiet = cmd_opts.quiet
137         self.verbose = cmd_opts.verbose
138         self.summary = summary
139         self.dn = dn.replace("${DOMAIN_DN}", self.con.base_dn)
140         self.dn = self.dn.replace("CN=${DOMAIN_NETBIOS}", "CN=%s" % self.con.domain_netbios)
141         for x in self.con.server_names:
142             self.dn = self.dn.replace("CN=${SERVERNAME}", "CN=%s" % x)
143         self.attributes = self.con.get_attributes(self.dn)
144         # Attributes that are considered always to be different e.g based on timestamp etc.
145         #
146         # One domain - two domain controllers
147         self.ignore_attributes =  [
148                 # Default Naming Context
149                 "lastLogon", "lastLogoff", "badPwdCount", "logonCount", "badPasswordTime", "modifiedCount",
150                 "operatingSystemVersion","oEMInformation",
151                 # Configuration Naming Context
152                 "repsFrom", "dSCorePropagationData", "msExchServer1HighestUSN",
153                 "replUpToDateVector", "repsTo", "whenChanged", "uSNChanged", "uSNCreated",
154                 # Schema Naming Context
155                 "prefixMap",]
156         self.dn_attributes = []
157         self.domain_attributes = []
158         self.servername_attributes = []
159         self.netbios_attributes = []
160         self.other_attributes = []
161         # Two domains - two domain controllers
162
163         if self.two_domains:
164             self.ignore_attributes +=  [
165                 "objectCategory", "objectGUID", "objectSid", "whenCreated", "pwdLastSet", "uSNCreated", "creationTime",
166                 "modifiedCount", "priorSetTime", "rIDManagerReference", "gPLink", "ipsecNFAReference",
167                 "fRSPrimaryMember", "fSMORoleOwner", "masteredBy", "ipsecOwnersReference", "wellKnownObjects",
168                 "badPwdCount", "ipsecISAKMPReference", "ipsecFilterReference", "msDs-masteredBy", "lastSetTime",
169                 "ipsecNegotiationPolicyReference", "subRefs", "gPCFileSysPath", "accountExpires", "invocationId",
170                 # After Exchange preps
171                 "targetAddress", "msExchMailboxGuid", "siteFolderGUID"]
172             #
173             # Attributes that contain the unique DN tail part e.g. 'DC=samba,DC=org'
174             self.dn_attributes = [
175                 "distinguishedName", "defaultObjectCategory", "member", "memberOf", "siteList", "nCName",
176                 "homeMDB", "homeMTA", "interSiteTopologyGenerator", "serverReference",
177                 "msDS-HasInstantiatedNCs", "hasMasterNCs", "msDS-hasMasterNCs", "msDS-HasDomainNCs", "dMDLocation",
178                 "msDS-IsDomainFor", "rIDSetReferences", "serverReferenceBL",
179                 # After Exchange preps
180                 "msExchHomeRoutingGroup", "msExchResponsibleMTAServer", "siteFolderServer", "msExchRoutingMasterDN",
181                 "msExchRoutingGroupMembersBL", "homeMDBBL", "msExchHomePublicMDB", "msExchOwningServer", "templateRoots",
182                 "addressBookRoots", "msExchPolicyRoots", "globalAddressList", "msExchOwningPFTree",
183                 "msExchResponsibleMTAServerBL", "msExchOwningPFTreeBL",]
184             self.dn_attributes = [x.upper() for x in self.dn_attributes]
185             #
186             # Attributes that contain the Domain name e.g. 'samba.org'
187             self.domain_attributes = [
188                 "proxyAddresses", "mail", "userPrincipalName", "msExchSmtpFullyQualifiedDomainName",
189                 "dnsHostName", "networkAddress", "dnsRoot", "servicePrincipalName",]
190             self.domain_attributes = [x.upper() for x in self.domain_attributes]
191             #
192             # May contain DOMAIN_NETBIOS and SERVERNAME
193             self.servername_attributes = [ "distinguishedName", "name", "CN", "sAMAccountName", "dNSHostName",
194                 "servicePrincipalName", "rIDSetReferences", "serverReference", "serverReferenceBL",
195                 "msDS-IsDomainFor", "interSiteTopologyGenerator",]
196             self.servername_attributes = [x.upper() for x in self.servername_attributes]
197             #
198             self.netbios_attributes = [ "servicePrincipalName", "CN", "distinguishedName", "nETBIOSName", "name",]
199             self.netbios_attributes = [x.upper() for x in self.netbios_attributes]
200             #
201             self.other_attributes = [ "name", "DC",]
202             self.other_attributes = [x.upper() for x in self.other_attributes]
203         #
204         self.ignore_attributes = [x.upper() for x in self.ignore_attributes]
205
206     def log(self, msg):
207         """
208         Log on the screen if there is no --quiet oprion set
209         """
210         if not self.quiet:
211             print msg
212
213     def fix_dn(self, s):
214         res = "%s" % s
215         if not self.two_domains:
216             return res
217         if res.upper().endswith(self.con.base_dn.upper()):
218             res = res[:len(res)-len(self.con.base_dn)] + "${DOMAIN_DN}"
219         return res
220
221     def fix_domain_name(self, s):
222         res = "%s" % s
223         if not self.two_domains:
224             return res
225         res = res.replace(self.con.domain_name.lower(), self.con.domain_name.upper())
226         res = res.replace(self.con.domain_name.upper(), "${DOMAIN_NAME}")
227         return res
228
229     def fix_domain_netbios(self, s):
230         res = "%s" % s
231         if not self.two_domains:
232             return res
233         res = res.replace(self.con.domain_netbios.lower(), self.con.domain_netbios.upper())
234         res = res.replace(self.con.domain_netbios.upper(), "${DOMAIN_NETBIOS}")
235         return res
236
237     def fix_server_name(self, s):
238         res = "%s" % s
239         if not self.two_domains or len(self.con.server_names) > 1:
240             return res
241         for x in self.con.server_names:
242             res = res.upper().replace(x, "${SERVERNAME}")
243         return res
244
245     def __eq__(self, other):
246         res = ""
247         self.unique_attrs = []
248         self.df_value_attrs = []
249         other.unique_attrs = []
250         if self.attributes.keys() != other.attributes.keys():
251             #
252             title = 4*" " + "Attributes found only in %s:" % self.con.host
253             for x in self.attributes.keys():
254                 if not x in other.attributes.keys() and \
255                 not x.upper() in [q.upper() for q in other.ignore_attributes]:
256                     if title:
257                         res += title + "\n"
258                         title = None
259                     res += 8*" " + x + "\n"
260                     self.unique_attrs.append(x)
261             #
262             title = 4*" " + "Attributes found only in %s:" % other.con.host
263             for x in other.attributes.keys():
264                 if not x in self.attributes.keys() and \
265                 not x.upper() in [q.upper() for q in self.ignore_attributes]:
266                     if title:
267                         res += title + "\n"
268                         title = None
269                     res += 8*" " + x + "\n"
270                     other.unique_attrs.append(x)
271         #
272         missing_attrs = [x.upper() for x in self.unique_attrs]
273         missing_attrs += [x.upper() for x in other.unique_attrs]
274         title = 4*" " + "Difference in attribute values:"
275         for x in self.attributes.keys():
276             if x.upper() in self.ignore_attributes or x.upper() in missing_attrs:
277                 continue
278             if isinstance(self.attributes[x], list) and isinstance(other.attributes[x], list):
279                 self.attributes[x] = sorted(self.attributes[x])
280                 other.attributes[x] = sorted(other.attributes[x])
281             if self.attributes[x] != other.attributes[x]:
282                 p = None
283                 q = None
284                 m = None
285                 n = None
286                 # First check if the difference can be fixed but shunting the first part
287                 # of the DomainHostName e.g. 'mysamba4.test.local' => 'mysamba4'
288                 if x.upper() in self.other_attributes:
289                     p = [self.con.domain_name.split(".")[0] == j for j in self.attributes[x]]
290                     q = [other.con.domain_name.split(".")[0] == j for j in other.attributes[x]]
291                     if p == q:
292                         continue
293                 # Attribute values that are list that contain DN based values that may differ
294                 elif x.upper() in self.dn_attributes:
295                     m = p
296                     n = q
297                     if not p and not q:
298                         m = self.attributes[x]
299                         n = other.attributes[x]
300                     p = [self.fix_dn(j) for j in m]
301                     q = [other.fix_dn(j) for j in n]
302                     if p == q:
303                         continue
304                 # Attributes that contain the Domain name in them
305                 if x.upper() in self.domain_attributes:
306                     m = p
307                     n = q
308                     if not p and not q:
309                         m = self.attributes[x]
310                         n = other.attributes[x]
311                     p = [self.fix_domain_name(j) for j in m]
312                     q = [other.fix_domain_name(j) for j in n]
313                     if p == q:
314                         continue
315                 #
316                 if x.upper() in self.servername_attributes:
317                     # Attributes with SERVERNAME
318                     m = p
319                     n = q
320                     if not p and not q:
321                         m = self.attributes[x]
322                         n = other.attributes[x]
323                     p = [self.fix_server_name(j) for j in m]
324                     q = [other.fix_server_name(j) for j in n]
325                     if p == q:
326                         continue
327                 #
328                 if x.upper() in self.netbios_attributes:
329                     # Attributes with NETBIOS Domain name
330                     m = p
331                     n = q
332                     if not p and not q:
333                         m = self.attributes[x]
334                         n = other.attributes[x]
335                     p = [self.fix_domain_netbios(j) for j in m]
336                     q = [other.fix_domain_netbios(j) for j in n]
337                     if p == q:
338                         continue
339                 #
340                 if title:
341                     res += title + "\n"
342                     title = None
343                 if p and q:
344                     res += 8*" " + x + " => \n%s\n%s" % (p, q) + "\n"
345                 else:
346                     res += 8*" " + x + " => \n%s\n%s" % (self.attributes[x], other.attributes[x]) + "\n"
347                 self.df_value_attrs.append(x)
348         #
349         if self.unique_attrs + other.unique_attrs != []:
350             assert self.unique_attrs != other.unique_attrs
351         self.summary["unique_attrs"] += self.unique_attrs
352         self.summary["df_value_attrs"] += self.df_value_attrs
353         other.summary["unique_attrs"] += other.unique_attrs
354         other.summary["df_value_attrs"] += self.df_value_attrs # they are the same
355         #
356         self.screen_output = res[:-1]
357         other.screen_output = res[:-1]
358         #
359         return res == ""
360
361
362 class LDAPBundel(object):
363     def __init__(self, connection, context, cmd_opts, dn_list=None):
364         self.con = connection
365         self.cmd_opts = cmd_opts
366         self.two_domains = cmd_opts.two
367         self.quiet = cmd_opts.quiet
368         self.verbose = cmd_opts.verbose
369         self.summary = {}
370         self.summary["unique_attrs"] = []
371         self.summary["df_value_attrs"] = []
372         self.summary["known_ignored_dn"] = []
373         self.summary["abnormal_ignored_dn"] = []
374         if dn_list:
375             self.dn_list = dn_list
376         elif context.upper() in ["DOMAIN", "CONFIGURATION", "SCHEMA"]:
377             self.context = context.upper()
378             self.dn_list = self.get_dn_list(context)
379         else:
380             raise Exception("Unknown initialization data for LDAPBundel().")
381         counter = 0
382         while counter < len(self.dn_list) and self.two_domains:
383             # Use alias reference
384             tmp = self.dn_list[counter]
385             tmp = tmp[:len(tmp)-len(self.con.base_dn)] + "${DOMAIN_DN}"
386             tmp = tmp.replace("CN=%s" % self.con.domain_netbios, "CN=${DOMAIN_NETBIOS}")
387             if len(self.con.server_names) == 1:
388                 for x in self.con.server_names:
389                     tmp = tmp.replace("CN=%s" % x, "CN=${SERVERNAME}")
390             self.dn_list[counter] = tmp
391             counter += 1
392         self.dn_list = list(set(self.dn_list))
393         self.dn_list = sorted(self.dn_list)
394         self.size = len(self.dn_list)
395
396     def log(self, msg):
397         """
398         Log on the screen if there is no --quiet oprion set
399         """
400         if not self.quiet:
401             print msg
402
403     def update_size(self):
404         self.size = len(self.dn_list)
405         self.dn_list = sorted(self.dn_list)
406
407     def __eq__(self, other):
408         res = True
409         if self.size != other.size:
410             self.log( "\n* DN lists have different size: %s != %s" % (self.size, other.size) )
411             res = False
412         #
413         title= "\n* DNs found only in %s:" % self.con.host
414         for x in self.dn_list:
415             if not x.upper() in [q.upper() for q in other.dn_list]:
416                 if title:
417                     self.log( title )
418                     title = None
419                     res = False
420                 self.log( 4*" " + x )
421                 self.dn_list[self.dn_list.index(x)] = ""
422         self.dn_list = [x for x in self.dn_list if x]
423         #
424         title= "\n* DNs found only in %s:" % other.con.host
425         for x in other.dn_list:
426             if not x.upper() in [q.upper() for q in self.dn_list]:
427                 if title:
428                     self.log( title )
429                     title = None
430                     res = False
431                 self.log( 4*" " + x )
432                 other.dn_list[other.dn_list.index(x)] = ""
433         other.dn_list = [x for x in other.dn_list if x]
434         #
435         self.update_size()
436         other.update_size()
437         assert self.size == other.size
438         assert sorted([x.upper() for x in self.dn_list]) == sorted([x.upper() for x in other.dn_list])
439         self.log( "\n* Objets to be compared: %s" % self.size )
440
441         index = 0
442         while index < self.size:
443             skip = False
444             try:
445                 object1 = LDAPObject(connection=self.con,
446                         dn=self.dn_list[index],
447                         summary=self.summary,
448                         cmd_opts = self.cmd_opts)
449             except LdbError, (ERR_NO_SUCH_OBJECT, _):
450                 self.log( "\n!!! Object not found: %s" % self.dn_list[index] )
451                 skip = True
452             try:
453                 object2 = LDAPObject(connection=other.con,
454                         dn=other.dn_list[index],
455                         summary=other.summary,
456                         cmd_opts = self.cmd_opts)
457             except LdbError, (ERR_NO_SUCH_OBJECT, _):
458                 self.log( "\n!!! Object not found: %s" % other.dn_list[index] )
459                 skip = True
460             if skip:
461                 index += 1
462                 continue
463             if object1 == object2:
464                 if self.verbose:
465                     self.log( "\nComparing:" )
466                     self.log( "'%s' [%s]" % (object1.dn, object1.con.host) )
467                     self.log( "'%s' [%s]" % (object2.dn, object2.con.host) )
468                     self.log( 4*" " + "OK" )
469             else:
470                 self.log( "\nComparing:" )
471                 self.log( "'%s' [%s]" % (object1.dn, object1.con.host) )
472                 self.log( "'%s' [%s]" % (object2.dn, object2.con.host) )
473                 self.log( object1.screen_output )
474                 self.log( 4*" " + "FAILED" )
475                 res = False
476             self.summary = object1.summary
477             other.summary = object2.summary
478             index += 1
479         #
480         return res
481
482     def get_dn_list(self, context):
483         """ Query LDAP server about the DNs of certain naming self.con.ext Domain (or Default), Configuration, Schema.
484             Parse all DNs and filter those that are 'strange' or abnormal.
485         """
486         if context.upper() == "DOMAIN":
487             search_base = "%s" % self.con.base_dn
488         elif context.upper() == "CONFIGURATION":
489             search_base = "CN=Configuration,%s" % self.con.base_dn
490         elif context.upper() == "SCHEMA":
491             search_base = "CN=Schema,CN=Configuration,%s" % self.con.base_dn
492
493         dn_list = []
494         res = self.con.ldb.search(base=search_base, scope=SCOPE_SUBTREE, attrs=["dn"])
495         for x in res:
496            dn_list.append(x["dn"].get_linearized())
497
498         #
499         global summary
500         #
501         return dn_list
502
503     def print_summary(self):
504         self.summary["unique_attrs"] = list(set(self.summary["unique_attrs"]))
505         self.summary["df_value_attrs"] = list(set(self.summary["df_value_attrs"]))
506         #
507         if self.summary["unique_attrs"]:
508             self.log( "\nAttributes found only in %s:" % self.con.host )
509             self.log( "".join([str("\n" + 4*" " + x) for x in self.summary["unique_attrs"]]) )
510         #
511         if self.summary["df_value_attrs"]:
512             self.log( "\nAttributes with different values:" )
513             self.log( "".join([str("\n" + 4*" " + x) for x in self.summary["df_value_attrs"]]) )
514             self.summary["df_value_attrs"] = []
515
516 ###
517
518 if __name__ == "__main__":
519     parser = OptionParser("ldapcmp [options] domain|configuration|schema")
520     sambaopts = options.SambaOptions(parser)
521     parser.add_option_group(sambaopts)
522     credopts = options.CredentialsOptionsDouble(parser)
523     parser.add_option_group(credopts)
524
525     parser.add_option("", "--host", dest="host",
526                               help="IP of the first LDAP server",)
527     parser.add_option("", "--host2", dest="host2",
528                               help="IP of the second LDAP server",)
529     parser.add_option("-w", "--two", dest="two", action="store_true", default=False,
530                               help="Hosts are in two different domains",)
531     parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
532                               help="Do not print anything but relay on just exit code",)
533     parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
534                               help="Print all DN pairs that have been compared",)
535     (opts, args) = parser.parse_args()
536
537     lp = sambaopts.get_loadparm()
538     creds = credopts.get_credentials(lp)
539     creds2 = credopts.get_credentials2(lp)
540     if creds2.is_anonymous():
541         creds2 = creds
542
543     if creds.is_anonymous():
544         parser.error("You must supply at least one username/password pair")
545
546     if not (len(args) == 1 and args[0].upper() in ["DOMAIN", "CONFIGURATION", "SCHEMA"]):
547         parser.error("Incorrect arguments")
548
549     if opts.verbose and opts.quiet:
550         parser.error("You cannot set --verbose and --quiet together")
551
552     con1 = LDAPBase(opts.host, opts, creds, lp)
553     assert len(con1.base_dn) > 0
554
555     con2 = LDAPBase(opts.host2, opts, creds2, lp)
556     assert len(con2.base_dn) > 0
557
558     b1 = LDAPBundel(con1, context=args[0], cmd_opts=opts)
559     b2 = LDAPBundel(con2, context=args[0], cmd_opts=opts)
560
561     if b1 == b2:
562         if not opts.quiet:
563             print "\n* Final result: SUCCESS"
564         status = 0
565     else:
566         if not opts.quiet:
567             print "\n* Final result: FAILURE"
568             print "\nSUMMARY"
569             print "---------"
570         status = -1
571
572     assert len(b1.summary["df_value_attrs"]) == len(b2.summary["df_value_attrs"])
573     b2.summary["df_value_attrs"] = []
574
575     if not opts.quiet:
576         b1.print_summary()
577         b2.print_summary()
578
579     sys.exit(status)