s4:kdc: adjust formatting of samba_kdc_update_pac() documentation
[Samba.git] / python / samba / netcmd / visualize.py
blob689d577ff8749bf792abe855a2194202222e9a86
1 # Visualisation tools
3 # Copyright (C) Andrew Bartlett 2015, 2018
5 # by Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 import os
21 import sys
22 from collections import defaultdict
23 import subprocess
24 import tempfile
25 import samba.getopt as options
26 from samba import dsdb
27 from samba import nttime2unix
28 from samba.netcmd import Command, SuperCommand, CommandError, Option
29 from samba.samdb import SamDB
30 from samba.graph import dot_graph
31 from samba.graph import distance_matrix, COLOUR_SETS
32 from samba.graph import full_matrix
33 from samba.colour import is_colour_wanted
35 from ldb import SCOPE_BASE, SCOPE_SUBTREE, LdbError
36 import time
37 import re
38 from samba.kcc import KCC, ldif_import_export
39 from samba.kcc.kcc_utils import KCCError
40 from samba.uptodateness import (
41 get_partition_maps,
42 get_partition,
43 get_own_cursor,
44 get_utdv,
45 get_utdv_edges,
46 get_utdv_distances,
47 get_utdv_max_distance,
48 get_kcc_and_dsas,
51 COMMON_OPTIONS = [
52 Option("-H", "--URL", help="LDB URL for database or target server",
53 type=str, metavar="URL", dest="H"),
54 Option("-o", "--output", help="write here (default stdout)",
55 type=str, metavar="FILE", default=None),
56 Option("--distance", help="Distance matrix graph output (default)",
57 dest='format', const='distance', action='store_const'),
58 Option("--utf8", help="Use utf-8 Unicode characters",
59 action='store_true'),
60 Option("--color-scheme", help=("use this colour scheme "
61 "(implies --color=yes)"),
62 choices=list(COLOUR_SETS.keys())),
63 Option("-S", "--shorten-names",
64 help="don't print long common suffixes",
65 action='store_true', default=False),
66 Option("-r", "--talk-to-remote", help="query other DCs' databases",
67 action='store_true', default=False),
68 Option("--no-key", help="omit the explanatory key",
69 action='store_false', default=True, dest='key'),
72 DOT_OPTIONS = [
73 Option("--dot", help="Graphviz dot output", dest='format',
74 const='dot', action='store_const'),
75 Option("--xdot", help="attempt to call Graphviz xdot", dest='format',
76 const='xdot', action='store_const'),
79 TEMP_FILE = '__temp__'
82 class GraphCommand(Command):
83 """Base class for graphing commands"""
85 synopsis = "%prog [options]"
86 takes_optiongroups = {
87 "sambaopts": options.SambaOptions,
88 "versionopts": options.VersionOptions,
89 "credopts": options.CredentialsOptions,
91 takes_options = COMMON_OPTIONS + DOT_OPTIONS
92 takes_args = ()
94 def get_db(self, H, sambaopts, credopts):
95 lp = sambaopts.get_loadparm()
96 creds = credopts.get_credentials(lp, fallback_machine=True)
97 samdb = SamDB(url=H, credentials=creds, lp=lp)
98 return samdb
100 def write(self, s, fn=None, suffix='.dot'):
101 """Decide whether we're dealing with a filename, a tempfile, or
102 stdout, and write accordingly.
104 :param s: the string to write
105 :param fn: a destination
106 :param suffix: suffix, if destination is a tempfile
108 If fn is None or "-", write to stdout.
109 If fn is visualize.TEMP_FILE, write to a temporary file
110 Otherwise fn should be a filename to write to.
112 if fn is None or fn == '-':
113 # we're just using stdout (a.k.a self.outf)
114 print(s, file=self.outf)
115 return
117 if fn is TEMP_FILE:
118 fd, fn = tempfile.mkstemp(prefix='samba-tool-visualise',
119 suffix=suffix)
120 f = open(fn, 'w')
121 os.close(fd)
122 else:
123 f = open(fn, 'w')
125 f.write(s)
126 f.close()
127 return fn
129 def calc_output_format(self, format, output):
130 """Heuristics to work out what output format was wanted."""
131 if not format:
132 # They told us nothing! We have to work it out for ourselves.
133 if output and output.lower().endswith('.dot'):
134 return 'dot'
135 else:
136 return 'distance'
138 if format == 'xdot':
139 return 'dot'
141 return format
143 def call_xdot(self, s, output):
144 if output is None:
145 fn = self.write(s, TEMP_FILE)
146 else:
147 fn = self.write(s, output)
148 xdot = os.environ.get('SAMBA_TOOL_XDOT_PATH', '/usr/bin/xdot')
149 subprocess.call([xdot, fn])
150 os.remove(fn)
152 def calc_distance_color_scheme(self, color_scheme, output):
153 """Heuristics to work out the colour scheme for distance matrices.
154 Returning None means no colour, otherwise it should be a colour
155 from graph.COLOUR_SETS"""
156 if color_scheme is not None:
157 # --color-scheme implies --color=yes for *this* purpose.
158 return color_scheme
160 if output in ('-', None):
161 output = self.outf
163 want_colour = is_colour_wanted(output, hint=self.requested_colour)
164 if not want_colour:
165 return None
167 # if we got to here, we are using colour according to the
168 # --color/NO_COLOR rules, but no colour scheme has been
169 # specified, so we choose some defaults.
170 if '256color' in os.environ.get('TERM', ''):
171 return 'xterm-256color-heatmap'
172 return 'ansi'
175 def get_dnstr_site(dn):
176 """Helper function for sorting and grouping DNs by site, if
177 possible."""
178 m = re.search(r'CN=Servers,CN=\s*([^,]+)\s*,CN=Sites', dn)
179 if m:
180 return m.group(1)
181 # Oh well, let it sort by DN
182 return dn
185 def get_dnstrlist_site(t):
186 """Helper function for sorting and grouping lists of (DN, ...) tuples
187 by site, if possible."""
188 return get_dnstr_site(t[0])
191 def colour_hash(x):
192 """Generate a randomish but consistent darkish colour based on the
193 given object."""
194 from hashlib import md5
195 tmp_str = str(x)
196 if isinstance(tmp_str, str):
197 tmp_str = tmp_str.encode('utf8')
198 c = int(md5(tmp_str).hexdigest()[:6], base=16) & 0x7f7f7f
199 return '#%06x' % c
202 class cmd_reps(GraphCommand):
203 "repsFrom/repsTo from every DSA"
205 takes_options = COMMON_OPTIONS + DOT_OPTIONS + [
206 Option("-p", "--partition", help="restrict to this partition",
207 default=None),
210 def run(self, H=None, output=None, shorten_names=False,
211 key=True, talk_to_remote=False,
212 sambaopts=None, credopts=None, versionopts=None,
213 mode='self', partition=None, color_scheme=None,
214 utf8=None, format=None, xdot=False):
215 # We use the KCC libraries in readonly mode to get the
216 # replication graph.
217 lp = sambaopts.get_loadparm()
218 creds = credopts.get_credentials(lp, fallback_machine=True)
219 local_kcc, dsas = get_kcc_and_dsas(H, lp, creds)
220 unix_now = local_kcc.unix_now
222 partition = get_partition(local_kcc.samdb, partition)
224 # nc_reps is an autovivifying dictionary of dictionaries of lists.
225 # nc_reps[partition]['current' | 'needed'] is a list of
226 # (dsa dn string, repsFromTo object) pairs.
227 nc_reps = defaultdict(lambda: defaultdict(list))
229 guid_to_dnstr = {}
231 # We run a new KCC for each DSA even if we aren't talking to
232 # the remote, because after kcc.run (or kcc.list_dsas) the kcc
233 # ends up in a messy state.
234 for dsa_dn in dsas:
235 kcc = KCC(unix_now, readonly=True)
236 if talk_to_remote:
237 res = local_kcc.samdb.search(dsa_dn,
238 scope=SCOPE_BASE,
239 attrs=["dNSHostName"])
240 dns_name = str(res[0]["dNSHostName"][0])
241 print("Attempting to contact ldap://%s (%s)" %
242 (dns_name, dsa_dn),
243 file=sys.stderr)
244 try:
245 kcc.load_samdb("ldap://%s" % dns_name, lp, creds)
246 except KCCError as e:
247 print("Could not contact ldap://%s (%s)" % (dns_name, e),
248 file=sys.stderr)
249 continue
251 kcc.run(H, lp, creds)
252 else:
253 kcc.load_samdb(H, lp, creds)
254 kcc.run(H, lp, creds, forced_local_dsa=dsa_dn)
256 dsas_from_here = set(kcc.list_dsas())
257 if dsas != dsas_from_here:
258 print("found extra DSAs:", file=sys.stderr)
259 for dsa in (dsas_from_here - dsas):
260 print(" %s" % dsa, file=sys.stderr)
261 print("missing DSAs (known locally, not by %s):" % dsa_dn,
262 file=sys.stderr)
263 for dsa in (dsas - dsas_from_here):
264 print(" %s" % dsa, file=sys.stderr)
266 for remote_dn in dsas_from_here:
267 if mode == 'others' and remote_dn == dsa_dn:
268 continue
269 elif mode == 'self' and remote_dn != dsa_dn:
270 continue
272 remote_dsa = kcc.get_dsa('CN=NTDS Settings,' + remote_dn)
273 kcc.translate_ntdsconn(remote_dsa)
274 guid_to_dnstr[str(remote_dsa.dsa_guid)] = remote_dn
275 # get_reps_tables() returns two dictionaries mapping
276 # dns to NCReplica objects
277 c, n = remote_dsa.get_rep_tables()
278 for part, rep in c.items():
279 if partition is None or part == partition:
280 nc_reps[part]['current'].append((dsa_dn, rep))
281 for part, rep in n.items():
282 if partition is None or part == partition:
283 nc_reps[part]['needed'].append((dsa_dn, rep))
285 all_edges = {'needed': {'to': [], 'from': []},
286 'current': {'to': [], 'from': []}}
288 short_partitions, long_partitions = get_partition_maps(local_kcc.samdb)
290 for partname, part in nc_reps.items():
291 for state, edgelists in all_edges.items():
292 for dsa_dn, rep in part[state]:
293 short_name = long_partitions.get(partname, partname)
294 for r in rep.rep_repsFrom:
295 edgelists['from'].append(
296 (dsa_dn,
297 guid_to_dnstr[str(r.source_dsa_obj_guid)],
298 short_name))
299 for r in rep.rep_repsTo:
300 edgelists['to'].append(
301 (guid_to_dnstr[str(r.source_dsa_obj_guid)],
302 dsa_dn,
303 short_name))
305 # Here we have the set of edges. From now it is a matter of
306 # interpretation and presentation.
308 if self.calc_output_format(format, output) == 'distance':
309 color_scheme = self.calc_distance_color_scheme(color_scheme,
310 output)
311 header_strings = {
312 'from': "RepsFrom objects for %s",
313 'to': "RepsTo objects for %s",
315 for state, edgelists in all_edges.items():
316 for direction, items in edgelists.items():
317 part_edges = defaultdict(list)
318 for src, dest, part in items:
319 part_edges[part].append((src, dest))
320 for part, edges in part_edges.items():
321 s = distance_matrix(None, edges,
322 utf8=utf8,
323 colour=color_scheme,
324 shorten_names=shorten_names,
325 generate_key=key,
326 grouping_function=get_dnstr_site)
328 s = "\n%s\n%s" % (header_strings[direction] % part, s)
329 self.write(s, output)
330 return
332 edge_colours = []
333 edge_styles = []
334 dot_edges = []
335 dot_vertices = set()
336 used_colours = {}
337 key_set = set()
338 for state, edgelist in all_edges.items():
339 for direction, items in edgelist.items():
340 for src, dest, part in items:
341 colour = used_colours.setdefault((part),
342 colour_hash((part,
343 direction)))
344 linestyle = 'dotted' if state == 'needed' else 'solid'
345 arrow = 'open' if direction == 'to' else 'empty'
346 dot_vertices.add(src)
347 dot_vertices.add(dest)
348 dot_edges.append((src, dest))
349 edge_colours.append(colour)
350 style = 'style="%s"; arrowhead=%s' % (linestyle, arrow)
351 edge_styles.append(style)
352 key_set.add((part, 'reps' + direction.title(),
353 colour, style))
355 key_items = []
356 if key:
357 for part, direction, colour, linestyle in sorted(key_set):
358 key_items.append((False,
359 'color="%s"; %s' % (colour, linestyle),
360 "%s %s" % (part, direction)))
361 key_items.append((False,
362 'style="dotted"; arrowhead="open"',
363 "repsFromTo is needed"))
364 key_items.append((False,
365 'style="solid"; arrowhead="open"',
366 "repsFromTo currently exists"))
368 s = dot_graph(dot_vertices, dot_edges,
369 directed=True,
370 edge_colors=edge_colours,
371 edge_styles=edge_styles,
372 shorten_names=shorten_names,
373 key_items=key_items)
375 if format == 'xdot':
376 self.call_xdot(s, output)
377 else:
378 self.write(s, output)
381 class NTDSConn(object):
382 """Collects observation counts for NTDS connections, so we know
383 whether all DSAs agree."""
384 def __init__(self, src, dest):
385 self.observations = 0
386 self.src_attests = False
387 self.dest_attests = False
388 self.src = src
389 self.dest = dest
391 def attest(self, attester):
392 self.observations += 1
393 if attester == self.src:
394 self.src_attests = True
395 if attester == self.dest:
396 self.dest_attests = True
399 class cmd_ntdsconn(GraphCommand):
400 "Draw the NTDSConnection graph"
401 takes_options = COMMON_OPTIONS + DOT_OPTIONS + [
402 Option("--importldif", help="graph from samba_kcc generated ldif",
403 default=None),
406 def import_ldif_db(self, ldif, lp):
407 d = tempfile.mkdtemp(prefix='samba-tool-visualise')
408 fn = os.path.join(d, 'imported.ldb')
409 self._tmp_fn_to_delete = fn
410 samdb = ldif_import_export.ldif_to_samdb(fn, lp, ldif)
411 return fn
413 def run(self, H=None, output=None, shorten_names=False,
414 key=True, talk_to_remote=False,
415 sambaopts=None, credopts=None, versionopts=None,
416 color_scheme=None,
417 utf8=None, format=None, importldif=None,
418 xdot=False):
420 lp = sambaopts.get_loadparm()
421 if importldif is None:
422 creds = credopts.get_credentials(lp, fallback_machine=True)
423 else:
424 creds = None
425 H = self.import_ldif_db(importldif, lp)
427 local_kcc, dsas = get_kcc_and_dsas(H, lp, creds)
428 local_dsa_dn = local_kcc.my_dsa_dnstr.split(',', 1)[1]
429 vertices = set()
430 attested_edges = []
431 for dsa_dn in dsas:
432 if talk_to_remote:
433 res = local_kcc.samdb.search(dsa_dn,
434 scope=SCOPE_BASE,
435 attrs=["dNSHostName"])
436 dns_name = res[0]["dNSHostName"][0]
437 try:
438 samdb = self.get_db("ldap://%s" % dns_name, sambaopts,
439 credopts)
440 except LdbError as e:
441 print("Could not contact ldap://%s (%s)" % (dns_name, e),
442 file=sys.stderr)
443 continue
445 ntds_dn = samdb.get_dsServiceName()
446 dn = samdb.domain_dn()
447 else:
448 samdb = self.get_db(H, sambaopts, credopts)
449 ntds_dn = 'CN=NTDS Settings,' + dsa_dn
450 dn = dsa_dn
452 res = samdb.search(ntds_dn,
453 scope=SCOPE_BASE,
454 attrs=["msDS-isRODC"])
456 is_rodc = res[0]["msDS-isRODC"][0] == 'TRUE'
458 vertices.add((ntds_dn, 'RODC' if is_rodc else ''))
459 # XXX we could also look at schedule
460 res = samdb.search(dn,
461 scope=SCOPE_SUBTREE,
462 expression="(objectClass=nTDSConnection)",
463 attrs=['fromServer'],
464 # XXX can't be critical for ldif test
465 # controls=["search_options:1:2"],
466 controls=["search_options:0:2"],
469 for msg in res:
470 msgdn = str(msg.dn)
471 dest_dn = msgdn[msgdn.index(',') + 1:]
472 attested_edges.append((str(msg['fromServer'][0]),
473 dest_dn, ntds_dn))
475 if importldif and H == self._tmp_fn_to_delete:
476 os.remove(H)
477 os.rmdir(os.path.dirname(H))
479 # now we overlay all the graphs and generate styles accordingly
480 edges = {}
481 for src, dest, attester in attested_edges:
482 k = (src, dest)
483 if k in edges:
484 e = edges[k]
485 else:
486 e = NTDSConn(*k)
487 edges[k] = e
488 e.attest(attester)
490 vertices, rodc_status = zip(*sorted(vertices))
492 if self.calc_output_format(format, output) == 'distance':
493 color_scheme = self.calc_distance_color_scheme(color_scheme,
494 output)
495 colours = COLOUR_SETS[color_scheme]
496 c_header = colours.get('header', '')
497 c_reset = colours.get('reset', '')
499 epilog = []
500 if 'RODC' in rodc_status:
501 epilog.append('No outbound connections are expected from RODCs')
503 if not talk_to_remote:
504 # If we are not talking to remote servers, we list all
505 # the connections.
506 graph_edges = edges.keys()
507 title = 'NTDS Connections known to %s' % local_dsa_dn
509 else:
510 # If we are talking to the remotes, there are
511 # interesting cases we can discover. What matters most
512 # is that the destination (i.e. owner) knowns about
513 # the connection, but it would be worth noting if the
514 # source doesn't. Another strange situation could be
515 # when a DC thinks there is a connection elsewhere,
516 # but the computers allegedly involved don't believe
517 # it exists.
519 # With limited bandwidth in the table, we mark the
520 # edges known to the destination, and note the other
521 # cases in a list after the diagram.
522 graph_edges = []
523 source_denies = []
524 dest_denies = []
525 both_deny = []
526 for e, conn in edges.items():
527 if conn.dest_attests:
528 graph_edges.append(e)
529 if not conn.src_attests:
530 source_denies.append(e)
531 elif conn.src_attests:
532 dest_denies.append(e)
533 else:
534 both_deny.append(e)
536 title = 'NTDS Connections known to each destination DC'
538 if both_deny:
539 epilog.append('The following connections are alleged by '
540 'DCs other than the source and '
541 'destination:\n')
542 for e in both_deny:
543 epilog.append(' %s -> %s\n' % e)
544 if dest_denies:
545 epilog.append('The following connections are alleged by '
546 'DCs other than the destination but '
547 'including the source:\n')
548 for e in dest_denies:
549 epilog.append(' %s -> %s\n' % e)
550 if source_denies:
551 epilog.append('The following connections '
552 '(included in the chart) '
553 'are not known to the source DC:\n')
554 for e in source_denies:
555 epilog.append(' %s -> %s\n' % e)
557 s = distance_matrix(vertices, graph_edges,
558 utf8=utf8,
559 colour=color_scheme,
560 shorten_names=shorten_names,
561 generate_key=key,
562 grouping_function=get_dnstrlist_site,
563 row_comments=rodc_status)
565 epilog = ''.join(epilog)
566 if epilog:
567 epilog = '\n%sNOTES%s\n%s' % (c_header,
568 c_reset,
569 epilog)
571 self.write('\n%s\n\n%s\n%s' % (title,
573 epilog), output)
574 return
576 dot_edges = []
577 edge_colours = []
578 edge_styles = []
579 edge_labels = []
580 n_servers = len(dsas)
581 for k, e in sorted(edges.items()):
582 dot_edges.append(k)
583 if e.observations == n_servers or not talk_to_remote:
584 edge_colours.append('#000000')
585 edge_styles.append('')
586 elif e.dest_attests:
587 edge_styles.append('')
588 if e.src_attests:
589 edge_colours.append('#0000ff')
590 else:
591 edge_colours.append('#cc00ff')
592 elif e.src_attests:
593 edge_colours.append('#ff0000')
594 edge_styles.append('style=dashed')
595 else:
596 edge_colours.append('#ff0000')
597 edge_styles.append('style=dotted')
599 key_items = []
600 if key:
601 key_items.append((False,
602 'color="#000000"',
603 "NTDS Connection"))
604 for colour, desc in (('#0000ff', "missing from some DCs"),
605 ('#cc00ff', "missing from source DC")):
606 if colour in edge_colours:
607 key_items.append((False, 'color="%s"' % colour, desc))
609 for style, desc in (('style=dashed', "unknown to destination"),
610 ('style=dotted',
611 "unknown to source and destination")):
612 if style in edge_styles:
613 key_items.append((False,
614 'color="#ff0000; %s"' % style,
615 desc))
617 if talk_to_remote:
618 title = 'NTDS Connections'
619 else:
620 title = 'NTDS Connections known to %s' % local_dsa_dn
622 s = dot_graph(sorted(vertices), dot_edges,
623 directed=True,
624 title=title,
625 edge_colors=edge_colours,
626 edge_labels=edge_labels,
627 edge_styles=edge_styles,
628 shorten_names=shorten_names,
629 key_items=key_items)
631 if format == 'xdot':
632 self.call_xdot(s, output)
633 else:
634 self.write(s, output)
637 class cmd_uptodateness(GraphCommand):
638 """visualize uptodateness vectors"""
640 takes_options = COMMON_OPTIONS + [
641 Option("-p", "--partition", help="restrict to this partition",
642 default=None),
643 Option("--max-digits", default=3, type=int,
644 help="display this many digits of out-of-date-ness"),
647 def run(self, H=None, output=None, shorten_names=False,
648 key=True, talk_to_remote=False,
649 sambaopts=None, credopts=None, versionopts=None,
650 color_scheme=None,
651 utf8=False, format=None, importldif=None,
652 xdot=False, partition=None, max_digits=3):
653 if not talk_to_remote:
654 print("this won't work without talking to the remote servers "
655 "(use -r)", file=self.outf)
656 return
658 # We use the KCC libraries in readonly mode to get the
659 # replication graph.
660 lp = sambaopts.get_loadparm()
661 creds = credopts.get_credentials(lp, fallback_machine=True)
662 local_kcc, dsas = get_kcc_and_dsas(H, lp, creds)
663 self.samdb = local_kcc.samdb
664 partition = get_partition(self.samdb, partition)
666 short_partitions, long_partitions = get_partition_maps(self.samdb)
667 color_scheme = self.calc_distance_color_scheme(color_scheme,
668 output)
670 for part_name, part_dn in short_partitions.items():
671 if partition not in (part_dn, None):
672 continue # we aren't doing this partition
674 utdv_edges = get_utdv_edges(local_kcc, dsas, part_dn, lp, creds)
676 distances = get_utdv_distances(utdv_edges, dsas)
678 max_distance = get_utdv_max_distance(distances)
680 digits = min(max_digits, len(str(max_distance)))
681 if digits < 1:
682 digits = 1
683 c_scale = 10 ** digits
685 s = full_matrix(distances,
686 utf8=utf8,
687 colour=color_scheme,
688 shorten_names=shorten_names,
689 generate_key=key,
690 grouping_function=get_dnstr_site,
691 colour_scale=c_scale,
692 digits=digits,
693 ylabel='DC',
694 xlabel='out-of-date-ness')
696 self.write('\n%s\n\n%s' % (part_name, s), output)
699 class cmd_visualize(SuperCommand):
700 """Produces graphical representations of Samba network state."""
701 subcommands = {}
703 for k, v in globals().items():
704 if k.startswith('cmd_'):
705 subcommands[k[4:]] = v()