3e5b9501e3a030bf9c8dde4600ae1858be2c3685
[kai/samba-autobuild/.git] / python / samba / netcmd / visualize.py
1 # Visualisation tools
2 #
3 # Copyright (C) Andrew Bartlett 2015, 2018
4 #
5 # by Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
19
20 from __future__ import print_function
21
22 import os
23 import sys
24 from collections import defaultdict
25
26 import tempfile
27 import samba
28 import samba.getopt as options
29 from samba.netcmd import Command, SuperCommand, CommandError, Option
30 from samba.samdb import SamDB
31 from samba.graph import dot_graph
32 from samba.graph import distance_matrix, COLOUR_SETS
33 from ldb import SCOPE_BASE, SCOPE_SUBTREE, LdbError
34 import time
35 from samba.kcc import KCC
36 from samba.kcc.kcc_utils import KCCError
37
38 COMMON_OPTIONS = [
39     Option("-H", "--URL", help="LDB URL for database or target server",
40            type=str, metavar="URL", dest="H"),
41     Option("-o", "--output", help="write here (default stdout)",
42            type=str, metavar="FILE", default=None),
43     Option("--dot", help="Graphviz dot output", dest='format',
44            const='dot', action='store_const'),
45     Option("--distance", help="Distance matrix graph output (default)",
46            dest='format', const='distance', action='store_const'),
47     Option("--utf8", help="Use utf-8 Unicode characters",
48            action='store_true'),
49     Option("--color", help="use color (yes, no, auto)",
50            choices=['yes', 'no', 'auto']),
51     Option("--color-scheme", help=("use this colour scheme "
52                                    "(implies --color=yes)"),
53            choices=COLOUR_SETS.keys()),
54     Option("-S", "--shorten-names",
55            help="don't print long common suffixes",
56            action='store_true', default=False),
57     Option("-r", "--talk-to-remote", help="query other DCs' databases",
58            action='store_true', default=False),
59     Option("--no-key", help="omit the explanatory key",
60            action='store_false', default=True, dest='key'),
61 ]
62
63 TEMP_FILE = '__temp__'
64
65
66 class GraphCommand(Command):
67     """Base class for graphing commands"""
68
69     synopsis = "%prog [options]"
70     takes_optiongroups = {
71         "sambaopts": options.SambaOptions,
72         "versionopts": options.VersionOptions,
73         "credopts": options.CredentialsOptions,
74     }
75     takes_options = COMMON_OPTIONS
76     takes_args = ()
77
78     def get_db(self, H, sambaopts, credopts):
79         lp = sambaopts.get_loadparm()
80         creds = credopts.get_credentials(lp, fallback_machine=True)
81         samdb = SamDB(url=H, credentials=creds, lp=lp)
82         return samdb
83
84     def get_kcc_and_dsas(self, H, lp, creds):
85         """Get a readonly KCC object and the list of DSAs it knows about."""
86         unix_now = int(time.time())
87         kcc = KCC(unix_now, readonly=True)
88         kcc.load_samdb(H, lp, creds)
89
90         dsa_list = kcc.list_dsas()
91         dsas = set(dsa_list)
92         if len(dsas) != len(dsa_list):
93             print("There seem to be duplicate dsas", file=sys.stderr)
94
95         return kcc, dsas
96
97     def write(self, s, fn=None, suffix='.dot'):
98         """Decide whether we're dealing with a filename, a tempfile, or
99         stdout, and write accordingly.
100
101         :param s: the string to write
102         :param fn: a destination
103         :param suffix: suffix, if destination is a tempfile
104
105         If fn is None or "-", write to stdout.
106         If fn is visualize.TEMP_FILE, write to a temporary file
107         Otherwise fn should be a filename to write to.
108         """
109         if fn is None or fn == '-':
110             # we're just using stdout (a.k.a self.outf)
111             print(s, file=self.outf)
112             return
113
114         if fn is TEMP_FILE:
115             fd, fn = tempfile.mkstemp(prefix='samba-tool-visualise',
116                                       suffix=suffix)
117             f = open(fn, 'w')
118             os.close(fd)
119         else:
120             f = open(fn, 'w')
121
122         f.write(s)
123         f.close()
124         return fn
125
126     def calc_output_format(self, format, output):
127         """Heuristics to work out what output format was wanted."""
128         if not format:
129             # They told us nothing! We have to work it out for ourselves.
130             if output and output.lower().endswith('.dot'):
131                 return 'dot'
132             else:
133                 return 'distance'
134         return format
135
136     def calc_distance_color_scheme(self, color, color_scheme, output):
137         """Heuristics to work out the colour scheme for distance matrices.
138         Returning None means no colour, otherwise it sould be a colour
139         from graph.COLOUR_SETS"""
140         if color == 'no':
141             return None
142
143         if color == 'auto':
144             if isinstance(output, str) and output != '-':
145                 return None
146             if not hasattr(self.outf, 'isatty'):
147                 # not a real file, perhaps cStringIO in testing
148                 return None
149             if not self.outf.isatty():
150                 return None
151
152         if color_scheme is None:
153             if '256color' in os.environ.get('TERM', ''):
154                 return 'xterm-256color-heatmap'
155             return 'ansi'
156
157         return color_scheme
158
159
160 def colour_hash(x):
161     """Generate a randomish but consistent darkish colour based on the
162     given object."""
163     from hashlib import md5
164     c = int(md5(str(x)).hexdigest()[:6], base=16) & 0x7f7f7f
165     return '#%06x' % c
166
167
168 def get_partition_maps(samdb):
169     """Generate dictionaries mapping short partition names to the
170     appropriate DNs."""
171     base_dn = samdb.domain_dn()
172     short_to_long = {
173         "DOMAIN": base_dn,
174         "CONFIGURATION": str(samdb.get_config_basedn()),
175         "SCHEMA": "CN=Schema,%s" % samdb.get_config_basedn(),
176         "DNSDOMAIN": "DC=DomainDnsZones,%s" % base_dn,
177         "DNSFOREST": "DC=ForestDnsZones,%s" % base_dn
178     }
179
180     long_to_short = {}
181     for s, l in short_to_long.iteritems():
182         long_to_short[l] = s
183
184     return short_to_long, long_to_short
185
186
187 class cmd_reps(GraphCommand):
188     "repsFrom/repsTo from every DSA"
189
190     takes_options = COMMON_OPTIONS + [
191         Option("-p", "--partition", help="restrict to this partition",
192                default=None),
193     ]
194
195     def run(self, H=None, output=None, shorten_names=False,
196             key=True, talk_to_remote=False,
197             sambaopts=None, credopts=None, versionopts=None,
198             mode='self', partition=None, color=None, color_scheme=None,
199             utf8=None, format=None):
200         # We use the KCC libraries in readonly mode to get the
201         # replication graph.
202         lp = sambaopts.get_loadparm()
203         creds = credopts.get_credentials(lp, fallback_machine=True)
204         local_kcc, dsas = self.get_kcc_and_dsas(H, lp, creds)
205         unix_now = local_kcc.unix_now
206
207         # Allow people to say "--partition=DOMAIN" rather than
208         # "--partition=DC=blah,DC=..."
209         short_partitions, long_partitions = get_partition_maps(local_kcc.samdb)
210         if partition is not None:
211             partition = short_partitions.get(partition.upper(), partition)
212             if partition not in long_partitions:
213                 raise CommandError("unknown partition %s" % partition)
214
215         # nc_reps is an autovivifying dictionary of dictionaries of lists.
216         # nc_reps[partition]['current' | 'needed'] is a list of
217         # (dsa dn string, repsFromTo object) pairs.
218         nc_reps = defaultdict(lambda: defaultdict(list))
219
220         guid_to_dnstr = {}
221
222         # We run a new KCC for each DSA even if we aren't talking to
223         # the remote, because after kcc.run (or kcc.list_dsas) the kcc
224         # ends up in a messy state.
225         for dsa_dn in dsas:
226             kcc = KCC(unix_now, readonly=True)
227             if talk_to_remote:
228                 res = local_kcc.samdb.search(dsa_dn,
229                                              scope=SCOPE_BASE,
230                                              attrs=["dNSHostName"])
231                 dns_name = res[0]["dNSHostName"][0]
232                 print("Attempting to contact ldap://%s (%s)" %
233                       (dns_name, dsa_dn),
234                       file=sys.stderr)
235                 try:
236                     kcc.load_samdb("ldap://%s" % dns_name, lp, creds)
237                 except KCCError as e:
238                     print("Could not contact ldap://%s (%s)" % (dns_name, e),
239                           file=sys.stderr)
240                     continue
241
242                 kcc.run(H, lp, creds)
243             else:
244                 kcc.load_samdb(H, lp, creds)
245                 kcc.run(H, lp, creds, forced_local_dsa=dsa_dn)
246
247             dsas_from_here = set(kcc.list_dsas())
248             if dsas != dsas_from_here:
249                 print("found extra DSAs:", file=sys.stderr)
250                 for dsa in (dsas_from_here - dsas):
251                     print("   %s" % dsa, file=sys.stderr)
252                 print("missing DSAs (known locally, not by %s):" % dsa_dn,
253                       file=sys.stderr)
254                 for dsa in (dsas - dsas_from_here):
255                     print("   %s" % dsa, file=sys.stderr)
256
257             for remote_dn in dsas_from_here:
258                 if mode == 'others' and remote_dn == dsa_dn:
259                     continue
260                 elif mode == 'self' and remote_dn != dsa_dn:
261                     continue
262
263                 remote_dsa = kcc.get_dsa('CN=NTDS Settings,' + remote_dn)
264                 kcc.translate_ntdsconn(remote_dsa)
265                 guid_to_dnstr[str(remote_dsa.dsa_guid)] = remote_dn
266                 # get_reps_tables() returns two dictionaries mapping
267                 # dns to NCReplica objects
268                 c, n = remote_dsa.get_rep_tables()
269                 for part, rep in c.iteritems():
270                     if partition is None or part == partition:
271                         nc_reps[part]['current'].append((dsa_dn, rep))
272                 for part, rep in n.iteritems():
273                     if partition is None or part == partition:
274                         nc_reps[part]['needed'].append((dsa_dn, rep))
275
276         all_edges = {'needed':  {'to': [], 'from': []},
277                      'current': {'to': [], 'from': []}}
278
279         for partname, part in nc_reps.iteritems():
280             for state, edgelists in all_edges.iteritems():
281                 for dsa_dn, rep in part[state]:
282                     short_name = long_partitions.get(partname, partname)
283                     for r in rep.rep_repsFrom:
284                         edgelists['from'].append(
285                             (dsa_dn,
286                              guid_to_dnstr[str(r.source_dsa_obj_guid)],
287                              short_name))
288                     for r in rep.rep_repsTo:
289                         edgelists['to'].append(
290                             (guid_to_dnstr[str(r.source_dsa_obj_guid)],
291                              dsa_dn,
292                              short_name))
293
294         # Here we have the set of edges. From now it is a matter of
295         # interpretation and presentation.
296
297         if self.calc_output_format(format, output) == 'distance':
298             color_scheme = self.calc_distance_color_scheme(color,
299                                                            color_scheme,
300                                                            output)
301             header_strings = {
302                 'from': "RepsFrom objects for %s",
303                 'to': "RepsTo objects for %s",
304             }
305             for state, edgelists in all_edges.iteritems():
306                 for direction, items in edgelists.iteritems():
307                     part_edges = defaultdict(list)
308                     for src, dest, part in items:
309                         part_edges[part].append((src, dest))
310                     for part, edges in part_edges.iteritems():
311                         s = distance_matrix(None, edges,
312                                             utf8=utf8,
313                                             colour=color_scheme,
314                                             shorten_names=shorten_names,
315                                             generate_key=key)
316
317                         s = "\n%s\n%s" % (header_strings[direction] % part, s)
318                         self.write(s, output)
319             return
320
321         edge_colours = []
322         edge_styles = []
323         dot_edges = []
324         dot_vertices = set()
325         used_colours = {}
326         key_set = set()
327         for state, edgelist in all_edges.iteritems():
328             for direction, items in edgelist.iteritems():
329                 for src, dest, part in items:
330                     colour = used_colours.setdefault((part),
331                                                      colour_hash((part,
332                                                                   direction)))
333                     linestyle = 'dotted' if state == 'needed' else 'solid'
334                     arrow = 'open' if direction == 'to' else 'empty'
335                     dot_vertices.add(src)
336                     dot_vertices.add(dest)
337                     dot_edges.append((src, dest))
338                     edge_colours.append(colour)
339                     style = 'style="%s"; arrowhead=%s' % (linestyle, arrow)
340                     edge_styles.append(style)
341                     key_set.add((part, 'reps' + direction.title(),
342                                  colour, style))
343
344         key_items = []
345         if key:
346             for part, direction, colour, linestyle in sorted(key_set):
347                 key_items.append((False,
348                                   'color="%s"; %s' % (colour, linestyle),
349                                   "%s %s" % (part, direction)))
350             key_items.append((False,
351                               'style="dotted"; arrowhead="open"',
352                               "repsFromTo is needed"))
353             key_items.append((False,
354                               'style="solid"; arrowhead="open"',
355                               "repsFromTo currently exists"))
356
357         s = dot_graph(dot_vertices, dot_edges,
358                       directed=True,
359                       edge_colors=edge_colours,
360                       edge_styles=edge_styles,
361                       shorten_names=shorten_names,
362                       key_items=key_items)
363
364         self.write(s, output)
365
366
367 class NTDSConn(object):
368     """Collects observation counts for NTDS connections, so we know
369     whether all DSAs agree."""
370     def __init__(self, src, dest):
371         self.observations = 0
372         self.src_attests = False
373         self.dest_attests = False
374         self.src = src
375         self.dest = dest
376
377     def attest(self, attester):
378         self.observations += 1
379         if attester == self.src:
380             self.src_attests = True
381         if attester == self.dest:
382             self.dest_attests = True
383
384
385 class cmd_ntdsconn(GraphCommand):
386     "Draw the NTDSConnection graph"
387     def run(self, H=None, output=None, shorten_names=False,
388             key=True, talk_to_remote=False,
389             sambaopts=None, credopts=None, versionopts=None,
390             color=None, color_scheme=None,
391             utf8=None, format=None):
392         lp = sambaopts.get_loadparm()
393         creds = credopts.get_credentials(lp, fallback_machine=True)
394         local_kcc, dsas = self.get_kcc_and_dsas(H, lp, creds)
395
396         vertices = set()
397         attested_edges = []
398         for dsa_dn in dsas:
399             if talk_to_remote:
400                 res = local_kcc.samdb.search(dsa_dn,
401                                              scope=SCOPE_BASE,
402                                              attrs=["dNSHostName"])
403                 dns_name = res[0]["dNSHostName"][0]
404                 try:
405                     samdb = self.get_db("ldap://%s" % dns_name, sambaopts,
406                                         credopts)
407                 except LdbError as e:
408                     print("Could not contact ldap://%s (%s)" % (dns_name, e),
409                           file=sys.stderr)
410                     continue
411
412                 ntds_dn = samdb.get_dsServiceName()
413                 dn = samdb.domain_dn()
414             else:
415                 samdb = self.get_db(H, sambaopts, credopts)
416                 ntds_dn = 'CN=NTDS Settings,' + dsa_dn
417                 dn = dsa_dn
418
419             vertices.add(ntds_dn)
420             # XXX we could also look at schedule
421             res = samdb.search(dn,
422                                scope=SCOPE_SUBTREE,
423                                expression="(objectClass=nTDSConnection)",
424                                attrs=['fromServer'],
425                                # XXX can't be critical for ldif test
426                                #controls=["search_options:1:2"],
427                                controls=["search_options:0:2"],
428             )
429
430             for msg in res:
431                 msgdn = str(msg.dn)
432                 dest_dn = msgdn[msgdn.index(',') + 1:]
433                 attested_edges.append((msg['fromServer'][0],
434                                        dest_dn, ntds_dn))
435
436         # now we overlay all the graphs and generate styles accordingly
437         edges = {}
438         for src, dest, attester in attested_edges:
439             k = (src, dest)
440             if k in edges:
441                 e = edges[k]
442             else:
443                 e = NTDSConn(*k)
444                 edges[k] = e
445             e.attest(attester)
446
447         if self.calc_output_format(format, output) == 'distance':
448             color_scheme = self.calc_distance_color_scheme(color,
449                                                            color_scheme,
450                                                            output)
451             if not talk_to_remote:
452                 # If we are not talking to remote servers, we list all
453                 # the connections.
454                 graph_edges = edges.keys()
455                 title = 'NTDS Connections known to %s' % dsa_dn
456                 epilog = ''
457
458             else:
459                 # If we are talking to the remotes, there are
460                 # interesting cases we can discover. What matters most
461                 # is that the destination (i.e. owner) knowns about
462                 # the connection, but it would be worth noting if the
463                 # source doesn't. Another strange situation could be
464                 # when a DC thinks there is a connection elsewhere,
465                 # but the computers allegedly involved don't believe
466                 # it exists.
467                 #
468                 # With limited bandwidth in the table, we mark the
469                 # edges known to the destination, and note the other
470                 # cases in a list after the diagram.
471                 graph_edges = []
472                 source_denies = []
473                 dest_denies = []
474                 both_deny = []
475                 for e, conn in edges.iteritems():
476                     if conn.dest_attests:
477                         graph_edges.append(e)
478                         if not conn.src_attests:
479                             source_denies.append(e)
480                     elif conn.src_attests:
481                         dest_denies.append(e)
482                     else:
483                         both_deny.append(e)
484
485                 title = 'NTDS Connections known to each destination DC'
486                 epilog = []
487                 if both_deny:
488                     epilog.append('The following connections are alleged by '
489                                   'DCs other than the source and '
490                                   'destination:\n')
491                     for e in both_deny:
492                         epilog.append('  %s -> %s\n' % e)
493                 if dest_denies:
494                     epilog.append('The following connections are alleged by '
495                                   'DCs other than the destination but '
496                                   'including the source:\n')
497                     for e in dest_denies:
498                         epilog.append('  %s -> %s\n' % e)
499                 if source_denies:
500                     epilog.append('The following connections '
501                                   '(included in the chart) '
502                                   'are not known to the source DC:\n')
503                     for e in source_denies:
504                         epilog.append('  %s -> %s\n' % e)
505                 epilog = ''.join(epilog)
506
507             s = distance_matrix(sorted(vertices), graph_edges,
508                                 utf8=utf8,
509                                 colour=color_scheme,
510                                 shorten_names=shorten_names,
511                                 generate_key=key)
512             self.write('\n%s\n%s\n%s' % (title, s, epilog), output)
513             return
514
515         dot_edges = []
516         edge_colours = []
517         edge_styles = []
518         edge_labels = []
519         n_servers = len(dsas)
520         for k, e in sorted(edges.iteritems()):
521             dot_edges.append(k)
522             if e.observations == n_servers or not talk_to_remote:
523                 edge_colours.append('#000000')
524                 edge_styles.append('')
525             elif e.dest_attests:
526                 edge_styles.append('')
527                 if e.src_attests:
528                     edge_colours.append('#0000ff')
529                 else:
530                     edge_colours.append('#cc00ff')
531             elif e.src_attests:
532                 edge_colours.append('#ff0000')
533                 edge_styles.append('style=dashed')
534             else:
535                 edge_colours.append('#ff0000')
536                 edge_styles.append('style=dotted')
537
538         key_items = []
539         if key:
540             key_items.append((False,
541                               'color="#000000"',
542                               "NTDS Connection"))
543             for colour, desc in (('#0000ff', "missing from some DCs"),
544                                  ('#cc00ff', "missing from source DC")):
545                 if colour in edge_colours:
546                     key_items.append((False, 'color="%s"' % colour, desc))
547
548             for style, desc in (('style=dashed', "unknown to destination"),
549                                 ('style=dotted',
550                                  "unknown to source and destination")):
551                 if style in edge_styles:
552                     key_items.append((False,
553                                       'color="#ff0000; %s"' % style,
554                                       desc))
555
556         if talk_to_remote:
557             title = 'NTDS Connections'
558         else:
559             title = 'NTDS Connections known to %s' % dsa_dn
560
561         s = dot_graph(sorted(vertices), dot_edges,
562                       directed=True,
563                       title=title,
564                       edge_colors=edge_colours,
565                       edge_labels=edge_labels,
566                       edge_styles=edge_styles,
567                       shorten_names=shorten_names,
568                       key_items=key_items)
569         self.write(s, output)
570
571
572 class cmd_visualize(SuperCommand):
573     """Produces graphical representations of Samba network state"""
574     subcommands = {}
575
576     for k, v in globals().iteritems():
577         if k.startswith('cmd_'):
578             subcommands[k[4:]] = v()