python/samba/netcmd/visualize.py

   1 # Visualisation tools
   2 #
   3 # Copyright (C) Andrew Bartlett 2015, 2018
   4 #
   5 # by Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 from __future__ import print_function
  21
  22 import os
  23 import sys
  24 from collections import defaultdict
  25
  26 import tempfile
  27 import samba
  28 import samba.getopt as options
  29 from samba.netcmd import Command, SuperCommand, CommandError, Option
  30 from samba.samdb import SamDB
  31 from samba.graph import dot_graph
  32 from samba.graph import distance_matrix, COLOUR_SETS
  33 from ldb import SCOPE_BASE, SCOPE_SUBTREE, LdbError
  34 import time
  35 import re
  36 from samba.kcc import KCC, ldif_import_export
  37 from samba.kcc.kcc_utils import KCCError
  38 from samba.compat import text_type
  39
  40 COMMON_OPTIONS = [
  41     Option("-H", "--URL", help="LDB URL for database or target server",
  42            type=str, metavar="URL", dest="H"),
  43     Option("-o", "--output", help="write here (default stdout)",
  44            type=str, metavar="FILE", default=None),
  45     Option("--dot", help="Graphviz dot output", dest='format',
  46            const='dot', action='store_const'),
  47     Option("--distance", help="Distance matrix graph output (default)",
  48            dest='format', const='distance', action='store_const'),
  49     Option("--utf8", help="Use utf-8 Unicode characters",
  50            action='store_true'),
  51     Option("--color", help="use color (yes, no, auto)",
  52            choices=['yes', 'no', 'auto']),
  53     Option("--color-scheme", help=("use this colour scheme "
  54                                    "(implies --color=yes)"),
  55            choices=list(COLOUR_SETS.keys())),
  56     Option("-S", "--shorten-names",
  57            help="don't print long common suffixes",
  58            action='store_true', default=False),
  59     Option("-r", "--talk-to-remote", help="query other DCs' databases",
  60            action='store_true', default=False),
  61     Option("--no-key", help="omit the explanatory key",
  62            action='store_false', default=True, dest='key'),
  63 ]
  64
  65 TEMP_FILE = '__temp__'
  66
  67
  68 class GraphCommand(Command):
  69     """Base class for graphing commands"""
  70
  71     synopsis = "%prog [options]"
  72     takes_optiongroups = {
  73         "sambaopts": options.SambaOptions,
  74         "versionopts": options.VersionOptions,
  75         "credopts": options.CredentialsOptions,
  76     }
  77     takes_options = COMMON_OPTIONS
  78     takes_args = ()
  79
  80     def get_db(self, H, sambaopts, credopts):
  81         lp = sambaopts.get_loadparm()
  82         creds = credopts.get_credentials(lp, fallback_machine=True)
  83         samdb = SamDB(url=H, credentials=creds, lp=lp)
  84         return samdb
  85
  86     def get_kcc_and_dsas(self, H, lp, creds):
  87         """Get a readonly KCC object and the list of DSAs it knows about."""
  88         unix_now = int(time.time())
  89         kcc = KCC(unix_now, readonly=True)
  90         kcc.load_samdb(H, lp, creds)
  91
  92         dsa_list = kcc.list_dsas()
  93         dsas = set(dsa_list)
  94         if len(dsas) != len(dsa_list):
  95             print("There seem to be duplicate dsas", file=sys.stderr)
  96
  97         return kcc, dsas
  98
  99     def write(self, s, fn=None, suffix='.dot'):
 100         """Decide whether we're dealing with a filename, a tempfile, or
 101         stdout, and write accordingly.
 102
 103         :param s: the string to write
 104         :param fn: a destination
 105         :param suffix: suffix, if destination is a tempfile
 106
 107         If fn is None or "-", write to stdout.
 108         If fn is visualize.TEMP_FILE, write to a temporary file
 109         Otherwise fn should be a filename to write to.
 110         """
 111         if fn is None or fn == '-':
 112             # we're just using stdout (a.k.a self.outf)
 113             print(s, file=self.outf)
 114             return
 115
 116         if fn is TEMP_FILE:
 117             fd, fn = tempfile.mkstemp(prefix='samba-tool-visualise',
 118                                       suffix=suffix)
 119             f = open(fn, 'w')
 120             os.close(fd)
 121         else:
 122             f = open(fn, 'w')
 123
 124         f.write(s)
 125         f.close()
 126         return fn
 127
 128     def calc_output_format(self, format, output):
 129         """Heuristics to work out what output format was wanted."""
 130         if not format:
 131             # They told us nothing! We have to work it out for ourselves.
 132             if output and output.lower().endswith('.dot'):
 133                 return 'dot'
 134             else:
 135                 return 'distance'
 136         return format
 137
 138     def calc_distance_color_scheme(self, color, color_scheme, output):
 139         """Heuristics to work out the colour scheme for distance matrices.
 140         Returning None means no colour, otherwise it sould be a colour
 141         from graph.COLOUR_SETS"""
 142         if color == 'no':
 143             return None
 144
 145         if color == 'auto':
 146             if isinstance(output, str) and output != '-':
 147                 return None
 148             if not hasattr(self.outf, 'isatty'):
 149                 # not a real file, perhaps cStringIO in testing
 150                 return None
 151             if not self.outf.isatty():
 152                 return None
 153
 154         if color_scheme is None:
 155             if '256color' in os.environ.get('TERM', ''):
 156                 return 'xterm-256color-heatmap'
 157             return 'ansi'
 158
 159         return color_scheme
 160
 161
 162 def get_dnstr_site(dn):
 163     """Helper function for sorting and grouping DNs by site, if
 164     possible."""
 165     m = re.search(r'CN=Servers,CN=\s*([^,]+)\s*,CN=Sites', dn)
 166     if m:
 167         return m.group(1)
 168     # Oh well, let it sort by DN
 169     return dn
 170
 171
 172 def colour_hash(x):
 173     """Generate a randomish but consistent darkish colour based on the
 174     given object."""
 175     from hashlib import md5
 176     tmp_str = str(x)
 177     if isinstance(tmp_str, text_type):
 178         tmp_str = tmp_str.encode('utf8')
 179     c = int(md5(tmp_str).hexdigest()[:6], base=16) & 0x7f7f7f
 180     return '#%06x' % c
 181
 182
 183 def get_partition_maps(samdb):
 184     """Generate dictionaries mapping short partition names to the
 185     appropriate DNs."""
 186     base_dn = samdb.domain_dn()
 187     short_to_long = {
 188         "DOMAIN": base_dn,
 189         "CONFIGURATION": str(samdb.get_config_basedn()),
 190         "SCHEMA": "CN=Schema,%s" % samdb.get_config_basedn(),
 191         "DNSDOMAIN": "DC=DomainDnsZones,%s" % base_dn,
 192         "DNSFOREST": "DC=ForestDnsZones,%s" % base_dn
 193     }
 194
 195     long_to_short = {}
 196     for s, l in short_to_long.items():
 197         long_to_short[l] = s
 198
 199     return short_to_long, long_to_short
 200
 201
 202 class cmd_reps(GraphCommand):
 203     "repsFrom/repsTo from every DSA"
 204
 205     takes_options = COMMON_OPTIONS + [
 206         Option("-p", "--partition", help="restrict to this partition",
 207                default=None),
 208     ]
 209
 210     def run(self, H=None, output=None, shorten_names=False,
 211             key=True, talk_to_remote=False,
 212             sambaopts=None, credopts=None, versionopts=None,
 213             mode='self', partition=None, color=None, color_scheme=None,
 214             utf8=None, format=None):
 215         # We use the KCC libraries in readonly mode to get the
 216         # replication graph.
 217         lp = sambaopts.get_loadparm()
 218         creds = credopts.get_credentials(lp, fallback_machine=True)
 219         local_kcc, dsas = self.get_kcc_and_dsas(H, lp, creds)
 220         unix_now = local_kcc.unix_now
 221
 222         # Allow people to say "--partition=DOMAIN" rather than
 223         # "--partition=DC=blah,DC=..."
 224         short_partitions, long_partitions = get_partition_maps(local_kcc.samdb)
 225         if partition is not None:
 226             partition = short_partitions.get(partition.upper(), partition)
 227             if partition not in long_partitions:
 228                 raise CommandError("unknown partition %s" % partition)
 229
 230         # nc_reps is an autovivifying dictionary of dictionaries of lists.
 231         # nc_reps[partition]['current' | 'needed'] is a list of
 232         # (dsa dn string, repsFromTo object) pairs.
 233         nc_reps = defaultdict(lambda: defaultdict(list))
 234
 235         guid_to_dnstr = {}
 236
 237         # We run a new KCC for each DSA even if we aren't talking to
 238         # the remote, because after kcc.run (or kcc.list_dsas) the kcc
 239         # ends up in a messy state.
 240         for dsa_dn in dsas:
 241             kcc = KCC(unix_now, readonly=True)
 242             if talk_to_remote:
 243                 res = local_kcc.samdb.search(dsa_dn,
 244                                              scope=SCOPE_BASE,
 245                                              attrs=["dNSHostName"])
 246                 dns_name = res[0]["dNSHostName"][0]
 247                 print("Attempting to contact ldap://%s (%s)" %
 248                       (dns_name, dsa_dn),
 249                       file=sys.stderr)
 250                 try:
 251                     kcc.load_samdb("ldap://%s" % dns_name, lp, creds)
 252                 except KCCError as e:
 253                     print("Could not contact ldap://%s (%s)" % (dns_name, e),
 254                           file=sys.stderr)
 255                     continue
 256
 257                 kcc.run(H, lp, creds)
 258             else:
 259                 kcc.load_samdb(H, lp, creds)
 260                 kcc.run(H, lp, creds, forced_local_dsa=dsa_dn)
 261
 262             dsas_from_here = set(kcc.list_dsas())
 263             if dsas != dsas_from_here:
 264                 print("found extra DSAs:", file=sys.stderr)
 265                 for dsa in (dsas_from_here - dsas):
 266                     print("   %s" % dsa, file=sys.stderr)
 267                 print("missing DSAs (known locally, not by %s):" % dsa_dn,
 268                       file=sys.stderr)
 269                 for dsa in (dsas - dsas_from_here):
 270                     print("   %s" % dsa, file=sys.stderr)
 271
 272             for remote_dn in dsas_from_here:
 273                 if mode == 'others' and remote_dn == dsa_dn:
 274                     continue
 275                 elif mode == 'self' and remote_dn != dsa_dn:
 276                     continue
 277
 278                 remote_dsa = kcc.get_dsa('CN=NTDS Settings,' + remote_dn)
 279                 kcc.translate_ntdsconn(remote_dsa)
 280                 guid_to_dnstr[str(remote_dsa.dsa_guid)] = remote_dn
 281                 # get_reps_tables() returns two dictionaries mapping
 282                 # dns to NCReplica objects
 283                 c, n = remote_dsa.get_rep_tables()
 284                 for part, rep in c.items():
 285                     if partition is None or part == partition:
 286                         nc_reps[part]['current'].append((dsa_dn, rep))
 287                 for part, rep in n.items():
 288                     if partition is None or part == partition:
 289                         nc_reps[part]['needed'].append((dsa_dn, rep))
 290
 291         all_edges = {'needed':  {'to': [], 'from': []},
 292                      'current': {'to': [], 'from': []}}
 293
 294         for partname, part in nc_reps.items():
 295             for state, edgelists in all_edges.items():
 296                 for dsa_dn, rep in part[state]:
 297                     short_name = long_partitions.get(partname, partname)
 298                     for r in rep.rep_repsFrom:
 299                         edgelists['from'].append(
 300                             (dsa_dn,
 301                              guid_to_dnstr[str(r.source_dsa_obj_guid)],
 302                              short_name))
 303                     for r in rep.rep_repsTo:
 304                         edgelists['to'].append(
 305                             (guid_to_dnstr[str(r.source_dsa_obj_guid)],
 306                              dsa_dn,
 307                              short_name))
 308
 309         # Here we have the set of edges. From now it is a matter of
 310         # interpretation and presentation.
 311
 312         if self.calc_output_format(format, output) == 'distance':
 313             color_scheme = self.calc_distance_color_scheme(color,
 314                                                            color_scheme,
 315                                                            output)
 316             header_strings = {
 317                 'from': "RepsFrom objects for %s",
 318                 'to': "RepsTo objects for %s",
 319             }
 320             for state, edgelists in all_edges.items():
 321                 for direction, items in edgelists.items():
 322                     part_edges = defaultdict(list)
 323                     for src, dest, part in items:
 324                         part_edges[part].append((src, dest))
 325                     for part, edges in part_edges.items():
 326                         s = distance_matrix(None, edges,
 327                                             utf8=utf8,
 328                                             colour=color_scheme,
 329                                             shorten_names=shorten_names,
 330                                             generate_key=key,
 331                                             grouping_function=get_dnstr_site)
 332
 333                         s = "\n%s\n%s" % (header_strings[direction] % part, s)
 334                         self.write(s, output)
 335             return
 336
 337         edge_colours = []
 338         edge_styles = []
 339         dot_edges = []
 340         dot_vertices = set()
 341         used_colours = {}
 342         key_set = set()
 343         for state, edgelist in all_edges.items():
 344             for direction, items in edgelist.items():
 345                 for src, dest, part in items:
 346                     colour = used_colours.setdefault((part),
 347                                                      colour_hash((part,
 348                                                                   direction)))
 349                     linestyle = 'dotted' if state == 'needed' else 'solid'
 350                     arrow = 'open' if direction == 'to' else 'empty'
 351                     dot_vertices.add(src)
 352                     dot_vertices.add(dest)
 353                     dot_edges.append((src, dest))
 354                     edge_colours.append(colour)
 355                     style = 'style="%s"; arrowhead=%s' % (linestyle, arrow)
 356                     edge_styles.append(style)
 357                     key_set.add((part, 'reps' + direction.title(),
 358                                  colour, style))
 359
 360         key_items = []
 361         if key:
 362             for part, direction, colour, linestyle in sorted(key_set):
 363                 key_items.append((False,
 364                                   'color="%s"; %s' % (colour, linestyle),
 365                                   "%s %s" % (part, direction)))
 366             key_items.append((False,
 367                               'style="dotted"; arrowhead="open"',
 368                               "repsFromTo is needed"))
 369             key_items.append((False,
 370                               'style="solid"; arrowhead="open"',
 371                               "repsFromTo currently exists"))
 372
 373         s = dot_graph(dot_vertices, dot_edges,
 374                       directed=True,
 375                       edge_colors=edge_colours,
 376                       edge_styles=edge_styles,
 377                       shorten_names=shorten_names,
 378                       key_items=key_items)
 379
 380         self.write(s, output)
 381
 382
 383 class NTDSConn(object):
 384     """Collects observation counts for NTDS connections, so we know
 385     whether all DSAs agree."""
 386     def __init__(self, src, dest):
 387         self.observations = 0
 388         self.src_attests = False
 389         self.dest_attests = False
 390         self.src = src
 391         self.dest = dest
 392
 393     def attest(self, attester):
 394         self.observations += 1
 395         if attester == self.src:
 396             self.src_attests = True
 397         if attester == self.dest:
 398             self.dest_attests = True
 399
 400
 401 class cmd_ntdsconn(GraphCommand):
 402     "Draw the NTDSConnection graph"
 403     takes_options = COMMON_OPTIONS + [
 404         Option("--importldif", help="graph from samba_kcc generated ldif",
 405                default=None),
 406     ]
 407
 408     def import_ldif_db(self, ldif, lp):
 409         d = tempfile.mkdtemp(prefix='samba-tool-visualise')
 410         fn = os.path.join(d, 'imported.ldb')
 411         self._tmp_fn_to_delete = fn
 412         samdb = ldif_import_export.ldif_to_samdb(fn, lp, ldif)
 413         return fn
 414
 415     def run(self, H=None, output=None, shorten_names=False,
 416             key=True, talk_to_remote=False,
 417             sambaopts=None, credopts=None, versionopts=None,
 418             color=None, color_scheme=None,
 419             utf8=None, format=None, importldif=None):
 420
 421         lp = sambaopts.get_loadparm()
 422         if importldif is None:
 423             creds = credopts.get_credentials(lp, fallback_machine=True)
 424         else:
 425             creds = None
 426             H = self.import_ldif_db(importldif, lp)
 427
 428         local_kcc, dsas = self.get_kcc_and_dsas(H, lp, creds)
 429         local_dsa_dn = local_kcc.my_dsa_dnstr.split(',', 1)[1]
 430         vertices = set()
 431         attested_edges = []
 432         for dsa_dn in dsas:
 433             if talk_to_remote:
 434                 res = local_kcc.samdb.search(dsa_dn,
 435                                              scope=SCOPE_BASE,
 436                                              attrs=["dNSHostName"])
 437                 dns_name = res[0]["dNSHostName"][0]
 438                 try:
 439                     samdb = self.get_db("ldap://%s" % dns_name, sambaopts,
 440                                         credopts)
 441                 except LdbError as e:
 442                     print("Could not contact ldap://%s (%s)" % (dns_name, e),
 443                           file=sys.stderr)
 444                     continue
 445
 446                 ntds_dn = samdb.get_dsServiceName()
 447                 dn = samdb.domain_dn()
 448             else:
 449                 samdb = self.get_db(H, sambaopts, credopts)
 450                 ntds_dn = 'CN=NTDS Settings,' + dsa_dn
 451                 dn = dsa_dn
 452
 453             res = samdb.search(ntds_dn,
 454                                scope=SCOPE_BASE,
 455                                attrs=["msDS-isRODC"])
 456
 457             is_rodc = res[0]["msDS-isRODC"][0] == 'TRUE'
 458
 459             vertices.add((ntds_dn, 'RODC' if is_rodc else ''))
 460             # XXX we could also look at schedule
 461             res = samdb.search(dn,
 462                                scope=SCOPE_SUBTREE,
 463                                expression="(objectClass=nTDSConnection)",
 464                                attrs=['fromServer'],
 465                                # XXX can't be critical for ldif test
 466                                #controls=["search_options:1:2"],
 467                                controls=["search_options:0:2"],
 468             )
 469
 470             for msg in res:
 471                 msgdn = str(msg.dn)
 472                 dest_dn = msgdn[msgdn.index(',') + 1:]
 473                 attested_edges.append((msg['fromServer'][0],
 474                                        dest_dn, ntds_dn))
 475
 476         if importldif and H == self._tmp_fn_to_delete:
 477             os.remove(H)
 478             os.rmdir(os.path.dirname(H))
 479
 480         # now we overlay all the graphs and generate styles accordingly
 481         edges = {}
 482         for src, dest, attester in attested_edges:
 483             k = (src, dest)
 484             if k in edges:
 485                 e = edges[k]
 486             else:
 487                 e = NTDSConn(*k)
 488                 edges[k] = e
 489             e.attest(attester)
 490
 491         vertices, rodc_status = zip(*sorted(vertices))
 492
 493         if self.calc_output_format(format, output) == 'distance':
 494             color_scheme = self.calc_distance_color_scheme(color,
 495                                                            color_scheme,
 496                                                            output)
 497             colours = COLOUR_SETS[color_scheme]
 498             c_header = colours.get('header', '')
 499             c_reset = colours.get('reset', '')
 500
 501             epilog = []
 502             if 'RODC' in rodc_status:
 503                 epilog.append('No outbound connections are expected from RODCs')
 504
 505             if not talk_to_remote:
 506                 # If we are not talking to remote servers, we list all
 507                 # the connections.
 508                 graph_edges = edges.keys()
 509                 title = 'NTDS Connections known to %s' % local_dsa_dn
 510
 511             else:
 512                 # If we are talking to the remotes, there are
 513                 # interesting cases we can discover. What matters most
 514                 # is that the destination (i.e. owner) knowns about
 515                 # the connection, but it would be worth noting if the
 516                 # source doesn't. Another strange situation could be
 517                 # when a DC thinks there is a connection elsewhere,
 518                 # but the computers allegedly involved don't believe
 519                 # it exists.
 520                 #
 521                 # With limited bandwidth in the table, we mark the
 522                 # edges known to the destination, and note the other
 523                 # cases in a list after the diagram.
 524                 graph_edges = []
 525                 source_denies = []
 526                 dest_denies = []
 527                 both_deny = []
 528                 for e, conn in edges.items():
 529                     if conn.dest_attests:
 530                         graph_edges.append(e)
 531                         if not conn.src_attests:
 532                             source_denies.append(e)
 533                     elif conn.src_attests:
 534                         dest_denies.append(e)
 535                     else:
 536                         both_deny.append(e)
 537
 538                 title = 'NTDS Connections known to each destination DC'
 539
 540                 if both_deny:
 541                     epilog.append('The following connections are alleged by '
 542                                   'DCs other than the source and '
 543                                   'destination:\n')
 544                     for e in both_deny:
 545                         epilog.append('  %s -> %s\n' % e)
 546                 if dest_denies:
 547                     epilog.append('The following connections are alleged by '
 548                                   'DCs other than the destination but '
 549                                   'including the source:\n')
 550                     for e in dest_denies:
 551                         epilog.append('  %s -> %s\n' % e)
 552                 if source_denies:
 553                     epilog.append('The following connections '
 554                                   '(included in the chart) '
 555                                   'are not known to the source DC:\n')
 556                     for e in source_denies:
 557                         epilog.append('  %s -> %s\n' % e)
 558
 559
 560             s = distance_matrix(vertices, graph_edges,
 561                                 utf8=utf8,
 562                                 colour=color_scheme,
 563                                 shorten_names=shorten_names,
 564                                 generate_key=key,
 565                                 grouping_function=get_dnstr_site,
 566                                 row_comments=rodc_status)
 567
 568             epilog = ''.join(epilog)
 569             if epilog:
 570                 epilog = '\n%sNOTES%s\n%s' % (c_header,
 571                                               c_reset,
 572                                               epilog)
 573
 574             self.write('\n%s\n\n%s\n%s' % (title,
 575                                            s,
 576                                            epilog), output)
 577             return
 578
 579         dot_edges = []
 580         edge_colours = []
 581         edge_styles = []
 582         edge_labels = []
 583         n_servers = len(dsas)
 584         for k, e in sorted(edges.items()):
 585             dot_edges.append(k)
 586             if e.observations == n_servers or not talk_to_remote:
 587                 edge_colours.append('#000000')
 588                 edge_styles.append('')
 589             elif e.dest_attests:
 590                 edge_styles.append('')
 591                 if e.src_attests:
 592                     edge_colours.append('#0000ff')
 593                 else:
 594                     edge_colours.append('#cc00ff')
 595             elif e.src_attests:
 596                 edge_colours.append('#ff0000')
 597                 edge_styles.append('style=dashed')
 598             else:
 599                 edge_colours.append('#ff0000')
 600                 edge_styles.append('style=dotted')
 601
 602         key_items = []
 603         if key:
 604             key_items.append((False,
 605                               'color="#000000"',
 606                               "NTDS Connection"))
 607             for colour, desc in (('#0000ff', "missing from some DCs"),
 608                                  ('#cc00ff', "missing from source DC")):
 609                 if colour in edge_colours:
 610                     key_items.append((False, 'color="%s"' % colour, desc))
 611
 612             for style, desc in (('style=dashed', "unknown to destination"),
 613                                 ('style=dotted',
 614                                  "unknown to source and destination")):
 615                 if style in edge_styles:
 616                     key_items.append((False,
 617                                       'color="#ff0000; %s"' % style,
 618                                       desc))
 619
 620         if talk_to_remote:
 621             title = 'NTDS Connections'
 622         else:
 623             title = 'NTDS Connections known to %s' % local_dsa_dn
 624
 625         s = dot_graph(sorted(vertices), dot_edges,
 626                       directed=True,
 627                       title=title,
 628                       edge_colors=edge_colours,
 629                       edge_labels=edge_labels,
 630                       edge_styles=edge_styles,
 631                       shorten_names=shorten_names,
 632                       key_items=key_items)
 633         self.write(s, output)
 634
 635
 636 class cmd_visualize(SuperCommand):
 637     """Produces graphical representations of Samba network state"""
 638     subcommands = {}
 639
 640     for k, v in globals().items():
 641         if k.startswith('cmd_'):
 642             subcommands[k[4:]] = v()