1 # Graph functions used by KCC intersite
3 # Copyright (C) Dave Craft 2011
4 # Copyright (C) Andrew Bartlett 2015
6 # Andrew Bartlett's alleged work performed by his underlings Douglas
7 # Bagnall and Garming Sam.
9 # This program is free software; you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation; either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 from samba.kcc.graph_utils import write_dot_file, verify_and_dot, verify_graph
26 from samba.ndr import ndr_pack
27 from samba.dcerpc import misc
29 from samba.kcc.debug import DEBUG, DEBUG_FN
31 MAX_DWORD = 2 ** 32 - 1
34 class ReplInfo(object):
35 """Represents information about replication
37 NTDSConnections use one representation a replication schedule, and
38 graph vertices use another. This is the Vertex one.
48 def total_schedule(schedule):
49 """Return the total number of 15 minute windows in which the schedule
50 is set to replicate in a week. If the schedule is None it is
51 assumed that the replication will happen in every 15 minute
54 This is essentially a bit population count.
58 return 84 * 8 # 84 bytes = 84 * 8 bits
68 def convert_schedule_to_repltimes(schedule):
69 """Convert NTDS Connection schedule to replTime schedule.
71 Schedule defined in MS-ADTS 6.1.4.5.2
72 ReplTimes defined in MS-DRSR 5.164.
74 "Schedule" has 168 bytes but only the lower nibble of each is
75 significant. There is one byte per hour. Bit 3 (0x08) represents
76 the first 15 minutes of the hour and bit 0 (0x01) represents the
77 last 15 minutes. The first byte presumably covers 12am - 1am
78 Sunday, though the spec doesn't define the start of a week.
80 "ReplTimes" has 84 bytes which are the 168 lower nibbles of
81 "Schedule" packed together. Thus each byte covers 2 hours. Bits 7
82 (i.e. 0x80) is the first 15 minutes and bit 0 is the last. The
83 first byte covers Sunday 12am - 2am (per spec).
85 Here we pack two elements of the NTDS Connection schedule slots
86 into one element of the replTimes list.
88 If no schedule appears in NTDS Connection then a default of 0x11
89 is set in each replTimes slot as per behaviour noted in a Windows
90 DC. That default would cause replication within the last 15
93 if schedule is None or schedule.dataArray[0] is None:
97 data = schedule.dataArray[0].slots
100 times.append((data[i * 2] & 0xF) << 4 | (data[i * 2 + 1] & 0xF))
105 # Returns true if schedule intersect
106 def combine_repl_info(info_a, info_b, info_c):
107 """Set a replInfo to be the intersection of two others
109 If there is any overlap in the replication times specified by the
110 first two parameters, the third replInfo parameter is set up with
111 that overlap, and True is returned. If there is no overlap, the
112 third parameter is unchanged and False is returned.
114 :param info_a: An input replInfo object
115 :param info_b: An input replInfo object
116 :param info_c: The result replInfo, set to the intersection of A and B
117 if the intersection is non-empty.
118 :return: True if info_c allows any replication at all, otherwise False
120 info_c.interval = max(info_a.interval, info_b.interval)
121 info_c.options = info_a.options & info_b.options
123 if info_a.schedule is None:
124 info_a.schedule = [0xFF] * 84
125 if info_b.schedule is None:
126 info_b.schedule = [0xFF] * 84
128 new_info = [a & b for a, b in zip(info_a.schedule, info_b.schedule)]
130 if not any(new_info):
133 info_c.schedule = new_info
135 # Truncate to MAX_DWORD
136 info_c.cost = info_a.cost + info_b.cost
137 if info_c.cost > MAX_DWORD:
138 info_c.cost = MAX_DWORD
143 def get_spanning_tree_edges(graph, my_site, label=None, verify=False,
145 """Find edges for the intersite graph
147 From MS-ADTS 6.2.2.3.4.4
149 :param graph: a kcc.kcc_utils.Graph object
150 :param my_site: the topology generator's site
151 :param label: a label for use in dot files and verification
152 :param verify: if True, try to verify that graph properties are correct
153 :param dot_file_dir: if not None, write Graphviz dot files here
155 # Phase 1: Run Dijkstra's to get a list of internal edges, which are
156 # just the shortest-paths connecting colored vertices
158 internal_edges = set()
160 for e_set in graph.edge_set:
162 for v in graph.vertices:
165 # All con_type in an edge set is the same
166 for e in e_set.edges:
167 edgeType = e.con_type
171 if verify or dot_file_dir is not None:
172 graph_edges = [(a.site.site_dnstr, b.site.site_dnstr)
175 *(itertools.combinations(edge.vertices, 2)
176 for edge in e_set.edges))]
177 graph_nodes = [v.site.site_dnstr for v in graph.vertices]
179 if dot_file_dir is not None:
180 write_dot_file('edgeset_%s' % (edgeType,), graph_edges,
181 vertices=graph_nodes, label=label)
184 verify_graph('spanning tree edge set %s' % edgeType,
185 graph_edges, vertices=graph_nodes,
186 properties=('complete', 'connected'),
189 # Run dijkstra's algorithm with just the red vertices as seeds
190 # Seed from the full replicas
191 dijkstra(graph, edgeType, False)
194 process_edge_set(graph, e_set, internal_edges)
196 # Run dijkstra's algorithm with red and black vertices as the seeds
197 # Seed from both full and partial replicas
198 dijkstra(graph, edgeType, True)
201 process_edge_set(graph, e_set, internal_edges)
203 # All vertices have root/component as itself
204 setup_vertices(graph)
205 process_edge_set(graph, None, internal_edges)
207 if verify or dot_file_dir is not None:
208 graph_edges = [(e.v1.site.site_dnstr, e.v2.site.site_dnstr)
209 for e in internal_edges]
210 graph_nodes = [v.site.site_dnstr for v in graph.vertices]
211 verify_properties = ('multi_edge_forest',)
212 verify_and_dot('prekruskal', graph_edges, graph_nodes, label=label,
213 properties=verify_properties, debug=DEBUG,
214 verify=verify, dot_file_dir=dot_file_dir)
216 # Phase 2: Run Kruskal's on the internal edges
217 output_edges, components = kruskal(graph, internal_edges)
219 # This recalculates the cost for the path connecting the
220 # closest red vertex. Ignoring types is fine because NO
221 # suboptimal edge should exist in the graph
222 dijkstra(graph, "EDGE_TYPE_ALL", False) # TODO rename
223 # Phase 3: Process the output
224 for v in graph.vertices:
228 v.dist_to_red = v.repl_info.cost
230 if verify or dot_file_dir is not None:
231 graph_edges = [(e.v1.site.site_dnstr, e.v2.site.site_dnstr)
232 for e in internal_edges]
233 graph_nodes = [v.site.site_dnstr for v in graph.vertices]
234 verify_properties = ('multi_edge_forest',)
235 verify_and_dot('postkruskal', graph_edges, graph_nodes,
236 label=label, properties=verify_properties,
237 debug=DEBUG, verify=verify,
238 dot_file_dir=dot_file_dir)
240 # Ensure only one-way connections for partial-replicas,
241 # and make sure they point the right way.
243 for edge in output_edges:
244 # We know these edges only have two endpoints because we made
247 if v.site is my_site or w.site is my_site:
248 if (((v.is_black() or w.is_black()) and
249 v.dist_to_red != MAX_DWORD)):
252 if w.dist_to_red < v.dist_to_red:
253 edge.vertices[:] = w, v
254 edge_list.append(edge)
256 if verify or dot_file_dir is not None:
257 graph_edges = [[x.site.site_dnstr for x in e.vertices]
259 #add the reverse edge if not directed.
260 graph_edges.extend([x.site.site_dnstr
261 for x in reversed(e.vertices)]
262 for e in edge_list if not e.directed)
263 graph_nodes = [x.site.site_dnstr for x in graph.vertices]
264 verify_properties = ()
265 verify_and_dot('post-one-way-partial', graph_edges, graph_nodes,
266 label=label, properties=verify_properties,
267 debug=DEBUG, verify=verify,
269 dot_file_dir=dot_file_dir)
271 # count the components
272 return edge_list, components
275 def create_edge(con_type, site_link, guid_to_vertex):
276 """Set up a MultiEdge for the intersite graph
278 A MultiEdge can have multiple vertices.
280 From MS-ADTS 6.2.2.3.4.4
282 :param con_type: a transport type GUID
283 :param site_link: a kcc.kcc_utils.SiteLink object
284 :param guid_to_vertex: a mapping between GUIDs and vertices
288 e.site_link = site_link
290 for site_guid in site_link.site_list:
291 if str(site_guid) in guid_to_vertex:
292 e.vertices.extend(guid_to_vertex.get(str(site_guid)))
293 e.repl_info.cost = site_link.cost
294 e.repl_info.options = site_link.options
295 e.repl_info.interval = site_link.interval
296 e.repl_info.schedule = convert_schedule_to_repltimes(site_link.schedule)
297 e.con_type = con_type
302 def create_auto_edge_set(graph, transport_guid):
303 """Set up an automatic MultiEdgeSet for the intersite graph
305 From within MS-ADTS 6.2.2.3.4.4
307 :param graph: the intersite graph object
308 :param transport_guid: a transport type GUID
309 :return: a MultiEdgeSet
311 e_set = MultiEdgeSet()
312 # use a NULL guid, not associated with a SiteLinkBridge object
313 e_set.guid = misc.GUID()
314 for site_link in graph.edges:
315 if site_link.con_type == transport_guid:
316 e_set.edges.append(site_link)
321 def create_edge_set(graph, transport, site_link_bridge):
322 # TODO not implemented - need to store all site link bridges
323 raise NotImplementedError("We don't create fancy edge sets")
326 def setup_vertices(graph):
327 for v in graph.vertices:
329 v.repl_info.cost = MAX_DWORD
331 v.component_id = None
337 v.repl_info.interval = 0
338 v.repl_info.options = 0xFFFFFFFF
339 v.repl_info.schedule = None # TODO highly suspicious
343 def dijkstra(graph, edge_type, include_black):
345 setup_dijkstra(graph, edge_type, include_black, queue)
346 while len(queue) > 0:
347 cost, guid, vertex = heapq.heappop(queue)
348 for edge in vertex.edges:
349 for v in edge.vertices:
351 # add new path from vertex to v
352 try_new_path(graph, queue, vertex, edge, v)
355 def setup_dijkstra(graph, edge_type, include_black, queue):
356 setup_vertices(graph)
357 for vertex in graph.vertices:
358 if vertex.is_white():
361 if (((vertex.is_black() and not include_black)
362 or edge_type not in vertex.accept_black
363 or edge_type not in vertex.accept_red_red)):
364 vertex.repl_info.cost = MAX_DWORD
365 vertex.root = None # NULL GUID
366 vertex.demoted = True # Demoted appears not to be used
368 heapq.heappush(queue, (vertex.repl_info.cost, vertex.guid, vertex))
371 def try_new_path(graph, queue, vfrom, edge, vto):
373 #This function combines the repl_info and checks is that there is
374 # a valid time frame for which replication can actually occur,
375 # despite being adequately connected
376 intersect = combine_repl_info(vfrom.repl_info, edge.repl_info, newRI)
378 # If the new path costs more than the current, then ignore the edge
379 if newRI.cost > vto.repl_info.cost:
382 if newRI.cost < vto.repl_info.cost and not intersect:
385 new_duration = total_schedule(newRI.schedule)
386 old_duration = total_schedule(vto.repl_info.schedule)
388 # Cheaper or longer schedule
389 if newRI.cost < vto.repl_info.cost or new_duration > old_duration:
390 vto.root = vfrom.root
391 vto.component_id = vfrom.component_id
392 vto.repl_info = newRI
393 heapq.heappush(queue, (vto.repl_info.cost, vto.guid, vto))
396 def check_demote_vertex(vertex, edge_type):
397 if vertex.is_white():
400 # Accepts neither red-red nor black edges, demote
401 if ((edge_type not in vertex.accept_black and
402 edge_type not in vertex.accept_red_red)):
403 vertex.repl_info.cost = MAX_DWORD
405 vertex.demoted = True # Demoted appears not to be used
408 def undemote_vertex(vertex):
409 if vertex.is_white():
412 vertex.repl_info.cost = 0
414 vertex.demoted = False
417 def process_edge_set(graph, e_set, internal_edges):
419 for edge in graph.edges:
420 for vertex in edge.vertices:
421 check_demote_vertex(vertex, edge.con_type)
422 process_edge(graph, edge, internal_edges)
423 for vertex in edge.vertices:
424 undemote_vertex(vertex)
426 for edge in e_set.edges:
427 process_edge(graph, edge, internal_edges)
430 def process_edge(graph, examine, internal_edges):
431 # Find the set of all vertices touches the edge to examine
433 for v in examine.vertices:
434 # Append a 4-tuple of color, repl cost, guid and vertex
435 vertices.append((v.color, v.repl_info.cost, v.ndrpacked_guid, v))
436 # Sort by color, lower
437 DEBUG("vertices is %s" % vertices)
440 color, cost, guid, bestv = vertices[0]
441 # Add to internal edges an edge from every colored vertex to bestV
442 for v in examine.vertices:
443 if v.component_id is None or v.root is None:
446 # Only add edge if valid inter-tree edge - needs a root and
447 # different components
448 if ((bestv.component_id is not None and
449 bestv.root is not None and
450 v.component_id is not None and
451 v.root is not None and
452 bestv.component_id != v.component_id)):
453 add_int_edge(graph, internal_edges, examine, bestv, v)
456 # Add internal edge, endpoints are roots of the vertices to pass in
457 # and are always red or black
458 def add_int_edge(graph, internal_edges, examine, v1, v2):
463 if root1.is_red() and root2.is_red():
467 if ((examine.con_type not in root1.accept_red_red
468 or examine.con_type not in root2.accept_red_red)):
470 elif (examine.con_type not in root1.accept_black
471 or examine.con_type not in root2.accept_black):
477 # Create the transitive replInfo for the two trees and this edge
478 if not combine_repl_info(v1.repl_info, v2.repl_info, ri):
480 # ri is now initialized
481 if not combine_repl_info(ri, examine.repl_info, ri2):
484 newIntEdge = InternalEdge(root1, root2, red_red, ri2, examine.con_type,
486 # Order by vertex guid
487 #XXX guid comparison using ndr_pack
488 if newIntEdge.v1.ndrpacked_guid > newIntEdge.v2.ndrpacked_guid:
489 newIntEdge.v1 = root2
490 newIntEdge.v2 = root1
492 internal_edges.add(newIntEdge)
495 def kruskal(graph, edges):
496 for v in graph.vertices:
499 components = set([x for x in graph.vertices if not x.is_white()])
502 # Sorted based on internal comparison function of internal edge
505 #XXX expected_num_tree_edges is never used
506 expected_num_tree_edges = 0 # TODO this value makes little sense
511 while index < len(edges): # TODO and num_components > 1
513 parent1 = find_component(e.v1)
514 parent2 = find_component(e.v2)
515 if parent1 is not parent2:
517 add_out_edge(graph, output_edges, e)
518 parent1.component_id = parent2
519 components.discard(parent1)
523 return output_edges, len(components)
526 def find_component(vertex):
527 if vertex.component_id is vertex:
531 while current.component_id is not current:
532 current = current.component_id
536 while current.component_id is not root:
537 n = current.component_id
538 current.component_id = root
544 def add_out_edge(graph, output_edges, e):
548 # This multi-edge is a 'real' edge with no GUID
551 ee.site_link = e.site_link
552 ee.vertices.append(v1)
553 ee.vertices.append(v2)
554 ee.con_type = e.e_type
555 ee.repl_info = e.repl_info
556 output_edges.append(ee)
562 def setup_graph(part, site_table, transport_guid, sitelink_table,
564 """Set up a GRAPH, populated with a VERTEX for each site
565 object, a MULTIEDGE for each siteLink object, and a
566 MUTLIEDGESET for each siteLinkBridge object (or implied
569 ::returns: a new graph
575 for site_guid, site in site_table.items():
576 vertex = Vertex(site, part)
577 vertex.guid = site_guid
578 vertex.ndrpacked_guid = ndr_pack(site.site_guid)
579 g.vertices.add(vertex)
580 guid_vertices = guid_to_vertex.setdefault(site_guid, [])
581 guid_vertices.append(vertex)
583 connected_vertices = set()
585 for site_link_dn, site_link in sitelink_table.items():
586 new_edge = create_edge(transport_guid, site_link,
588 connected_vertices.update(new_edge.vertices)
589 g.edges.add(new_edge)
591 # If 'Bridge all site links' is enabled and Win2k3 bridges required
593 # NTDSTRANSPORT_OPT_BRIDGES_REQUIRED 0x00000002
594 # No documentation for this however, ntdsapi.h appears to have:
595 # NTDSSETTINGS_OPT_W2K3_BRIDGES_REQUIRED = 0x00001000
597 g.edge_set.add(create_auto_edge_set(g, transport_guid))
599 # TODO get all site link bridges
600 for site_link_bridge in []:
601 g.edge_set.add(create_edge_set(g, transport_guid,
604 g.connected_vertices = connected_vertices
609 class VertexColor(object):
610 (red, black, white, unknown) = range(0, 4)
613 class Vertex(object):
614 """Class encapsulation of a Site Vertex in the
615 intersite topology replication algorithm
617 def __init__(self, site, part):
620 self.color = VertexColor.unknown
622 self.accept_red_red = []
623 self.accept_black = []
624 self.repl_info = ReplInfo()
627 self.component_id = self
632 def color_vertex(self):
633 """Color each vertex to indicate which kind of NC
636 # IF s contains one or more DCs with full replicas of the
638 # SET v.Color to COLOR.RED
639 # ELSEIF s contains one or more partial replicas of the NC
640 # SET v.Color to COLOR.BLACK
642 # SET v.Color to COLOR.WHITE
644 # set to minimum (no replica)
645 self.color = VertexColor.white
647 for dnstr, dsa in self.site.dsa_table.items():
648 rep = dsa.get_current_replica(self.part.nc_dnstr)
652 # We have a full replica which is the largest
654 if not rep.is_partial():
655 self.color = VertexColor.red
658 self.color = VertexColor.black
661 assert(self.color != VertexColor.unknown)
662 return (self.color == VertexColor.red)
665 assert(self.color != VertexColor.unknown)
666 return (self.color == VertexColor.black)
669 assert(self.color != VertexColor.unknown)
670 return (self.color == VertexColor.white)
673 class IntersiteGraph(object):
674 """Graph for representing the intersite"""
676 self.vertices = set()
678 self.edge_set = set()
679 # All vertices that are endpoints of edges
680 self.connected_vertices = None
683 class MultiEdgeSet(object):
684 """Defines a multi edge set"""
686 self.guid = 0 # objectGuid siteLinkBridge
690 class MultiEdge(object):
692 self.site_link = None # object siteLink
694 self.con_type = None # interSiteTransport GUID
695 self.repl_info = ReplInfo()
699 class InternalEdge(object):
700 def __init__(self, v1, v2, redred, repl, eType, site_link):
703 self.red_red = redred
704 self.repl_info = repl
706 self.site_link = site_link
708 def __eq__(self, other):
709 return not self < other and not other < self
711 def __ne__(self, other):
712 return self < other or other < self
714 def __gt__(self, other):
717 def __ge__(self, other):
718 return not self < other
720 def __le__(self, other):
721 return not other < self
723 # TODO compare options and interval
724 def __lt__(self, other):
725 if self.red_red != other.red_red:
728 if self.repl_info.cost != other.repl_info.cost:
729 return self.repl_info.cost < other.repl_info.cost
731 self_time = total_schedule(self.repl_info.schedule)
732 other_time = total_schedule(other.repl_info.schedule)
733 if self_time != other_time:
734 return self_time > other_time
736 #XXX guid comparison using ndr_pack
737 if self.v1.guid != other.v1.guid:
738 return self.v1.ndrpacked_guid < other.v1.ndrpacked_guid
740 if self.v2.guid != other.v2.guid:
741 return self.v2.ndrpacked_guid < other.v2.ndrpacked_guid
743 return self.e_type < other.e_type