1 # -*- encoding: utf-8 -*-
2 # Samba traffic replay and learning
4 # Copyright (C) Catalyst IT Ltd. 2017
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from __future__ import print_function, division
30 from collections import OrderedDict, Counter, defaultdict
31 from samba.emulate import traffic_packets
32 from samba.samdb import SamDB
34 from ldb import LdbError
35 from samba.dcerpc import ClientConnection
36 from samba.dcerpc import security, drsuapi, lsa
37 from samba.dcerpc import netlogon
38 from samba.dcerpc.netlogon import netr_Authenticator
39 from samba.dcerpc import srvsvc
40 from samba.dcerpc import samr
41 from samba.drs_utils import drs_DsBind
43 from samba.credentials import Credentials, DONT_USE_KERBEROS, MUST_USE_KERBEROS
44 from samba.auth import system_session
45 from samba.dsdb import (
47 UF_SERVER_TRUST_ACCOUNT,
48 UF_TRUSTED_FOR_DELEGATION
50 from samba.dcerpc.misc import SEC_CHAN_BDC
51 from samba import gensec
52 from samba import sd_utils
56 # we don't use None, because it complicates [de]serialisation
60 ('dns', '0'): 1.0, # query
61 ('smb', '0x72'): 1.0, # Negotiate protocol
62 ('ldap', '0'): 1.0, # bind
63 ('ldap', '3'): 1.0, # searchRequest
64 ('ldap', '2'): 1.0, # unbindRequest
66 ('dcerpc', '11'): 1.0, # bind
67 ('dcerpc', '14'): 1.0, # Alter_context
68 ('nbns', '0'): 1.0, # query
72 ('dns', '1'): 1.0, # response
73 ('ldap', '1'): 1.0, # bind response
74 ('ldap', '4'): 1.0, # search result
75 ('ldap', '5'): 1.0, # search done
77 ('dcerpc', '12'): 1.0, # bind_ack
78 ('dcerpc', '13'): 1.0, # bind_nak
79 ('dcerpc', '15'): 1.0, # Alter_context response
82 SKIPPED_PROTOCOLS = {"smb", "smb2", "browser", "smb_netlogon"}
85 WAIT_THRESHOLD = (1.0 / WAIT_SCALE)
86 NO_WAIT_LOG_TIME_RANGE = (-10, -3)
88 # DEBUG_LEVEL can be changed by scripts with -d
92 def debug(level, msg, *args):
93 """Print a formatted debug message to standard error.
96 :param level: The debug level, message will be printed if it is <= the
97 currently set debug level. The debug level can be set with
99 :param msg: The message to be logged, can contain C-Style format
101 :param args: The parameters required by the format specifiers
103 if level <= DEBUG_LEVEL:
105 print(msg, file=sys.stderr)
107 print(msg % tuple(args), file=sys.stderr)
110 def debug_lineno(*args):
111 """ Print an unformatted log message to stderr, contaning the line number
113 tb = traceback.extract_stack(limit=2)
114 print((" %s:" "\033[01;33m"
115 "%s " "\033[00m" % (tb[0][2], tb[0][1])), end=' ',
118 print(a, file=sys.stderr)
119 print(file=sys.stderr)
123 def random_colour_print():
124 """Return a function that prints a randomly coloured line to stderr"""
125 n = 18 + random.randrange(214)
126 prefix = "\033[38;5;%dm" % n
130 print("%s%s\033[00m" % (prefix, a), file=sys.stderr)
135 class FakePacketError(Exception):
139 class Packet(object):
140 """Details of a network packet"""
141 def __init__(self, timestamp, ip_protocol, stream_number, src, dest,
142 protocol, opcode, desc, extra):
144 self.timestamp = timestamp
145 self.ip_protocol = ip_protocol
146 self.stream_number = stream_number
149 self.protocol = protocol
153 if self.src < self.dest:
154 self.endpoints = (self.src, self.dest)
156 self.endpoints = (self.dest, self.src)
159 def from_line(self, line):
160 fields = line.rstrip('\n').split('\t')
171 timestamp = float(timestamp)
175 return Packet(timestamp, ip_protocol, stream_number, src, dest,
176 protocol, opcode, desc, extra)
178 def as_summary(self, time_offset=0.0):
179 """Format the packet as a traffic_summary line.
181 extra = '\t'.join(self.extra)
182 t = self.timestamp + time_offset
183 return (t, '%f\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' %
186 self.stream_number or '',
195 return ("%.3f: %d -> %d; ip %s; strm %s; prot %s; op %s; desc %s %s" %
196 (self.timestamp, self.src, self.dest, self.ip_protocol or '-',
197 self.stream_number, self.protocol, self.opcode, self.desc,
198 ('«' + ' '.join(self.extra) + '»' if self.extra else '')))
201 return "<Packet @%s>" % self
204 return self.__class__(self.timestamp,
214 def as_packet_type(self):
215 t = '%s:%s' % (self.protocol, self.opcode)
218 def client_score(self):
219 """A positive number means we think it is a client; a negative number
220 means we think it is a server. Zero means no idea. range: -1 to 1.
222 key = (self.protocol, self.opcode)
223 if key in CLIENT_CLUES:
224 return CLIENT_CLUES[key]
225 if key in SERVER_CLUES:
226 return -SERVER_CLUES[key]
229 def play(self, conversation, context):
230 """Send the packet over the network, if required.
232 Some packets are ignored, i.e. for protocols not handled,
233 server response messages, or messages that are generated by the
234 protocol layer associated with other packets.
236 fn_name = 'packet_%s_%s' % (self.protocol, self.opcode)
238 fn = getattr(traffic_packets, fn_name)
240 except AttributeError as e:
241 print("Conversation(%s) Missing handler %s" % \
242 (conversation.conversation_id, fn_name),
246 # Don't display a message for kerberos packets, they're not directly
247 # generated they're used to indicate kerberos should be used
248 if self.protocol != "kerberos":
249 debug(2, "Conversation(%s) Calling handler %s" %
250 (conversation.conversation_id, fn_name))
254 if fn(self, conversation, context):
255 # Only collect timing data for functions that generate
256 # network traffic, or fail
258 duration = end - start
259 print("%f\t%s\t%s\t%s\t%f\tTrue\t" %
260 (end, conversation.conversation_id, self.protocol,
261 self.opcode, duration))
262 except Exception as e:
264 duration = end - start
265 print("%f\t%s\t%s\t%s\t%f\tFalse\t%s" %
266 (end, conversation.conversation_id, self.protocol,
267 self.opcode, duration, e))
269 def __cmp__(self, other):
270 return self.timestamp - other.timestamp
272 def is_really_a_packet(self, missing_packet_stats=None):
273 """Is the packet one that can be ignored?
275 If so removing it will have no effect on the replay
277 if self.protocol in SKIPPED_PROTOCOLS:
278 # Ignore any packets for the protocols we're not interested in.
280 if self.protocol == "ldap" and self.opcode == '':
281 # skip ldap continuation packets
284 fn_name = 'packet_%s_%s' % (self.protocol, self.opcode)
285 fn = getattr(traffic_packets, fn_name, None)
287 print("missing packet %s" % fn_name, file=sys.stderr)
289 if fn is traffic_packets.null_packet:
294 class ReplayContext(object):
295 """State/Context for an individual conversation between an simulated client
303 badpassword_frequency=None,
304 prefer_kerberos=None,
313 self.ldap_connections = []
314 self.dcerpc_connections = []
315 self.lsarpc_connections = []
316 self.lsarpc_connections_named = []
317 self.drsuapi_connections = []
318 self.srvsvc_connections = []
319 self.samr_contexts = []
320 self.netlogon_connection = None
323 self.prefer_kerberos = prefer_kerberos
325 self.base_dn = base_dn
327 self.statsdir = statsdir
328 self.global_tempdir = tempdir
329 self.domain_sid = domain_sid
330 self.realm = lp.get('realm')
332 # Bad password attempt controls
333 self.badpassword_frequency = badpassword_frequency
334 self.last_lsarpc_bad = False
335 self.last_lsarpc_named_bad = False
336 self.last_simple_bind_bad = False
337 self.last_bind_bad = False
338 self.last_srvsvc_bad = False
339 self.last_drsuapi_bad = False
340 self.last_netlogon_bad = False
341 self.last_samlogon_bad = False
342 self.generate_ldap_search_tables()
343 self.next_conversation_id = itertools.count()
345 def generate_ldap_search_tables(self):
346 session = system_session()
348 db = SamDB(url="ldap://%s" % self.server,
349 session_info=session,
350 credentials=self.creds,
353 res = db.search(db.domain_dn(),
354 scope=ldb.SCOPE_SUBTREE,
355 controls=["paged_results:1:1000"],
358 # find a list of dns for each pattern
359 # e.g. CN,CN,CN,DC,DC
361 attribute_clue_map = {
367 pattern = ','.join(x.lstrip()[:2] for x in dn.split(',')).upper()
368 dns = dn_map.setdefault(pattern, [])
370 if dn.startswith('CN=NTDS Settings,'):
371 attribute_clue_map['invocationId'].append(dn)
373 # extend the map in case we are working with a different
374 # number of DC components.
375 # for k, v in self.dn_map.items():
376 # print >>sys.stderr, k, len(v)
378 for k in list(dn_map.keys()):
382 while p[-3:] == ',DC':
386 if p != k and p in dn_map:
387 print('dn_map collison %s %s' % (k, p),
390 dn_map[p] = dn_map[k]
393 self.attribute_clue_map = attribute_clue_map
395 def generate_process_local_config(self, account, conversation):
398 self.netbios_name = account.netbios_name
399 self.machinepass = account.machinepass
400 self.username = account.username
401 self.userpass = account.userpass
403 self.tempdir = mk_masked_dir(self.global_tempdir,
405 conversation.conversation_id)
407 self.lp.set("private dir", self.tempdir)
408 self.lp.set("lock dir", self.tempdir)
409 self.lp.set("state directory", self.tempdir)
410 self.lp.set("tls verify peer", "no_check")
412 # If the domain was not specified, check for the environment
414 if self.domain is None:
415 self.domain = os.environ["DOMAIN"]
417 self.remoteAddress = "/root/ncalrpc_as_system"
418 self.samlogon_dn = ("cn=%s,%s" %
419 (self.netbios_name, self.ou))
420 self.user_dn = ("cn=%s,%s" %
421 (self.username, self.ou))
423 self.generate_machine_creds()
424 self.generate_user_creds()
426 def with_random_bad_credentials(self, f, good, bad, failed_last_time):
427 """Execute the supplied logon function, randomly choosing the
430 Based on the frequency in badpassword_frequency randomly perform the
431 function with the supplied bad credentials.
432 If run with bad credentials, the function is re-run with the good
434 failed_last_time is used to prevent consecutive bad credential
435 attempts. So the over all bad credential frequency will be lower
436 than that requested, but not significantly.
438 if not failed_last_time:
439 if (self.badpassword_frequency > 0 and
440 random.random() < self.badpassword_frequency):
444 # Ignore any exceptions as the operation may fail
445 # as it's being performed with bad credentials
447 failed_last_time = True
449 failed_last_time = False
452 return (result, failed_last_time)
454 def generate_user_creds(self):
455 """Generate the conversation specific user Credentials.
457 Each Conversation has an associated user account used to simulate
458 any non Administrative user traffic.
460 Generates user credentials with good and bad passwords and ldap
461 simple bind credentials with good and bad passwords.
463 self.user_creds = Credentials()
464 self.user_creds.guess(self.lp)
465 self.user_creds.set_workstation(self.netbios_name)
466 self.user_creds.set_password(self.userpass)
467 self.user_creds.set_username(self.username)
468 self.user_creds.set_domain(self.domain)
469 if self.prefer_kerberos:
470 self.user_creds.set_kerberos_state(MUST_USE_KERBEROS)
472 self.user_creds.set_kerberos_state(DONT_USE_KERBEROS)
474 self.user_creds_bad = Credentials()
475 self.user_creds_bad.guess(self.lp)
476 self.user_creds_bad.set_workstation(self.netbios_name)
477 self.user_creds_bad.set_password(self.userpass[:-4])
478 self.user_creds_bad.set_username(self.username)
479 if self.prefer_kerberos:
480 self.user_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
482 self.user_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
484 # Credentials for ldap simple bind.
485 self.simple_bind_creds = Credentials()
486 self.simple_bind_creds.guess(self.lp)
487 self.simple_bind_creds.set_workstation(self.netbios_name)
488 self.simple_bind_creds.set_password(self.userpass)
489 self.simple_bind_creds.set_username(self.username)
490 self.simple_bind_creds.set_gensec_features(
491 self.simple_bind_creds.get_gensec_features() | gensec.FEATURE_SEAL)
492 if self.prefer_kerberos:
493 self.simple_bind_creds.set_kerberos_state(MUST_USE_KERBEROS)
495 self.simple_bind_creds.set_kerberos_state(DONT_USE_KERBEROS)
496 self.simple_bind_creds.set_bind_dn(self.user_dn)
498 self.simple_bind_creds_bad = Credentials()
499 self.simple_bind_creds_bad.guess(self.lp)
500 self.simple_bind_creds_bad.set_workstation(self.netbios_name)
501 self.simple_bind_creds_bad.set_password(self.userpass[:-4])
502 self.simple_bind_creds_bad.set_username(self.username)
503 self.simple_bind_creds_bad.set_gensec_features(
504 self.simple_bind_creds_bad.get_gensec_features() |
506 if self.prefer_kerberos:
507 self.simple_bind_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
509 self.simple_bind_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
510 self.simple_bind_creds_bad.set_bind_dn(self.user_dn)
512 def generate_machine_creds(self):
513 """Generate the conversation specific machine Credentials.
515 Each Conversation has an associated machine account.
517 Generates machine credentials with good and bad passwords.
520 self.machine_creds = Credentials()
521 self.machine_creds.guess(self.lp)
522 self.machine_creds.set_workstation(self.netbios_name)
523 self.machine_creds.set_secure_channel_type(SEC_CHAN_BDC)
524 self.machine_creds.set_password(self.machinepass)
525 self.machine_creds.set_username(self.netbios_name + "$")
526 self.machine_creds.set_domain(self.domain)
527 if self.prefer_kerberos:
528 self.machine_creds.set_kerberos_state(MUST_USE_KERBEROS)
530 self.machine_creds.set_kerberos_state(DONT_USE_KERBEROS)
532 self.machine_creds_bad = Credentials()
533 self.machine_creds_bad.guess(self.lp)
534 self.machine_creds_bad.set_workstation(self.netbios_name)
535 self.machine_creds_bad.set_secure_channel_type(SEC_CHAN_BDC)
536 self.machine_creds_bad.set_password(self.machinepass[:-4])
537 self.machine_creds_bad.set_username(self.netbios_name + "$")
538 if self.prefer_kerberos:
539 self.machine_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
541 self.machine_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
543 def get_matching_dn(self, pattern, attributes=None):
544 # If the pattern is an empty string, we assume ROOTDSE,
545 # Otherwise we try adding or removing DC suffixes, then
546 # shorter leading patterns until we hit one.
547 # e.g if there is no CN,CN,CN,CN,DC,DC
548 # we first try CN,CN,CN,CN,DC
549 # and CN,CN,CN,CN,DC,DC,DC
550 # then change to CN,CN,CN,DC,DC
551 # and as last resort we use the base_dn
552 attr_clue = self.attribute_clue_map.get(attributes)
554 return random.choice(attr_clue)
556 pattern = pattern.upper()
558 if pattern in self.dn_map:
559 return random.choice(self.dn_map[pattern])
560 # chop one off the front and try it all again.
561 pattern = pattern[3:]
565 def get_dcerpc_connection(self, new=False):
566 guid = '12345678-1234-abcd-ef00-01234567cffb' # RPC_NETLOGON UUID
567 if self.dcerpc_connections and not new:
568 return self.dcerpc_connections[-1]
569 c = ClientConnection("ncacn_ip_tcp:%s" % self.server,
571 self.dcerpc_connections.append(c)
574 def get_srvsvc_connection(self, new=False):
575 if self.srvsvc_connections and not new:
576 return self.srvsvc_connections[-1]
579 return srvsvc.srvsvc("ncacn_np:%s" % (self.server),
583 (c, self.last_srvsvc_bad) = \
584 self.with_random_bad_credentials(connect,
587 self.last_srvsvc_bad)
589 self.srvsvc_connections.append(c)
592 def get_lsarpc_connection(self, new=False):
593 if self.lsarpc_connections and not new:
594 return self.lsarpc_connections[-1]
597 binding_options = 'schannel,seal,sign'
598 return lsa.lsarpc("ncacn_ip_tcp:%s[%s]" %
599 (self.server, binding_options),
603 (c, self.last_lsarpc_bad) = \
604 self.with_random_bad_credentials(connect,
606 self.machine_creds_bad,
607 self.last_lsarpc_bad)
609 self.lsarpc_connections.append(c)
612 def get_lsarpc_named_pipe_connection(self, new=False):
613 if self.lsarpc_connections_named and not new:
614 return self.lsarpc_connections_named[-1]
617 return lsa.lsarpc("ncacn_np:%s" % (self.server),
621 (c, self.last_lsarpc_named_bad) = \
622 self.with_random_bad_credentials(connect,
624 self.machine_creds_bad,
625 self.last_lsarpc_named_bad)
627 self.lsarpc_connections_named.append(c)
630 def get_drsuapi_connection_pair(self, new=False, unbind=False):
631 """get a (drs, drs_handle) tuple"""
632 if self.drsuapi_connections and not new:
633 c = self.drsuapi_connections[-1]
637 binding_options = 'seal'
638 binding_string = "ncacn_ip_tcp:%s[%s]" %\
639 (self.server, binding_options)
640 return drsuapi.drsuapi(binding_string, self.lp, creds)
642 (drs, self.last_drsuapi_bad) = \
643 self.with_random_bad_credentials(connect,
646 self.last_drsuapi_bad)
648 (drs_handle, supported_extensions) = drs_DsBind(drs)
649 c = (drs, drs_handle)
650 self.drsuapi_connections.append(c)
653 def get_ldap_connection(self, new=False, simple=False):
654 if self.ldap_connections and not new:
655 return self.ldap_connections[-1]
657 def simple_bind(creds):
659 To run simple bind against Windows, we need to run
660 following commands in PowerShell:
662 Install-windowsfeature ADCS-Cert-Authority
663 Install-AdcsCertificationAuthority -CAType EnterpriseRootCA
667 return SamDB('ldaps://%s' % self.server,
671 def sasl_bind(creds):
672 return SamDB('ldap://%s' % self.server,
676 (samdb, self.last_simple_bind_bad) = \
677 self.with_random_bad_credentials(simple_bind,
678 self.simple_bind_creds,
679 self.simple_bind_creds_bad,
680 self.last_simple_bind_bad)
682 (samdb, self.last_bind_bad) = \
683 self.with_random_bad_credentials(sasl_bind,
688 self.ldap_connections.append(samdb)
691 def get_samr_context(self, new=False):
692 if not self.samr_contexts or new:
693 self.samr_contexts.append(
694 SamrContext(self.server, lp=self.lp, creds=self.creds))
695 return self.samr_contexts[-1]
697 def get_netlogon_connection(self):
699 if self.netlogon_connection:
700 return self.netlogon_connection
703 return netlogon.netlogon("ncacn_ip_tcp:%s[schannel,seal]" %
707 (c, self.last_netlogon_bad) = \
708 self.with_random_bad_credentials(connect,
710 self.machine_creds_bad,
711 self.last_netlogon_bad)
712 self.netlogon_connection = c
715 def guess_a_dns_lookup(self):
716 return (self.realm, 'A')
718 def get_authenticator(self):
719 auth = self.machine_creds.new_client_authenticator()
720 current = netr_Authenticator()
721 current.cred.data = [ord(x) for x in auth["credential"]]
722 current.timestamp = auth["timestamp"]
724 subsequent = netr_Authenticator()
725 return (current, subsequent)
728 class SamrContext(object):
729 """State/Context associated with a samr connection.
731 def __init__(self, server, lp=None, creds=None):
732 self.connection = None
734 self.domain_handle = None
735 self.domain_sid = None
736 self.group_handle = None
737 self.user_handle = None
743 def get_connection(self):
744 if not self.connection:
745 self.connection = samr.samr(
746 "ncacn_ip_tcp:%s[seal]" % (self.server),
748 credentials=self.creds)
750 return self.connection
752 def get_handle(self):
754 c = self.get_connection()
755 self.handle = c.Connect2(None, security.SEC_FLAG_MAXIMUM_ALLOWED)
759 class Conversation(object):
760 """Details of a converation between a simulated client and a server."""
761 conversation_id = None
763 def __init__(self, start_time=None, endpoints=None):
764 self.start_time = start_time
765 self.endpoints = endpoints
767 self.msg = random_colour_print()
768 self.client_balance = 0.0
770 def __cmp__(self, other):
771 if self.start_time is None:
772 if other.start_time is None:
775 if other.start_time is None:
777 return self.start_time - other.start_time
779 def add_packet(self, packet):
780 """Add a packet object to this conversation, making a local copy with
781 a conversation-relative timestamp."""
784 if self.start_time is None:
785 self.start_time = p.timestamp
787 if self.endpoints is None:
788 self.endpoints = p.endpoints
790 if p.endpoints != self.endpoints:
791 raise FakePacketError("Conversation endpoints %s don't match"
792 "packet endpoints %s" %
793 (self.endpoints, p.endpoints))
795 p.timestamp -= self.start_time
797 if p.src == p.endpoints[0]:
798 self.client_balance -= p.client_score()
800 self.client_balance += p.client_score()
802 if p.is_really_a_packet():
803 self.packets.append(p)
805 def add_short_packet(self, timestamp, protocol, opcode, extra,
807 """Create a packet from a timestamp, and 'protocol:opcode' pair, and a
808 (possibly empty) list of extra data. If client is True, assume
809 this packet is from the client to the server.
811 src, dest = self.guess_client_server()
813 src, dest = dest, src
814 key = (protocol, opcode)
815 desc = OP_DESCRIPTIONS[key] if key in OP_DESCRIPTIONS else ''
816 if protocol in IP_PROTOCOLS:
817 ip_protocol = IP_PROTOCOLS[protocol]
820 packet = Packet(timestamp - self.start_time, ip_protocol,
822 protocol, opcode, desc, extra)
823 # XXX we're assuming the timestamp is already adjusted for
825 # XXX should we adjust client balance for guessed packets?
826 if packet.src == packet.endpoints[0]:
827 self.client_balance -= packet.client_score()
829 self.client_balance += packet.client_score()
830 if packet.is_really_a_packet():
831 self.packets.append(packet)
834 return ("<Conversation %s %s starting %.3f %d packets>" %
835 (self.conversation_id, self.endpoints, self.start_time,
841 return iter(self.packets)
844 return len(self.packets)
846 def get_duration(self):
847 if len(self.packets) < 2:
849 return self.packets[-1].timestamp - self.packets[0].timestamp
851 def replay_as_summary_lines(self):
853 for p in self.packets:
854 lines.append(p.as_summary(self.start_time))
857 def replay_in_fork_with_delay(self, start, context=None, account=None):
858 """Fork a new process and replay the conversation.
860 def signal_handler(signal, frame):
861 """Signal handler closes standard out and error.
863 Triggered by a sigterm, ensures that the log messages are flushed
864 to disk and not lost.
871 now = time.time() - start
873 # we are replaying strictly in order, so it is safe to sleep
874 # in the main process if the gap is big enough. This reduces
875 # the number of concurrent threads, which allows us to make
877 if gap > 0.15 and False:
878 print("sleeping for %f in main process" % (gap - 0.1),
880 time.sleep(gap - 0.1)
881 now = time.time() - start
883 print("gap is now %f" % gap, file=sys.stderr)
885 self.conversation_id = next(context.next_conversation_id)
890 signal.signal(signal.SIGTERM, signal_handler)
891 # we must never return, or we'll end up running parts of the
892 # parent's clean-up code. So we work in a try...finally, and
893 # try to print any exceptions.
896 context.generate_process_local_config(account, self)
899 filename = os.path.join(context.statsdir, 'stats-conversation-%d' %
900 self.conversation_id)
902 sys.stdout = open(filename, 'w')
904 sleep_time = gap - SLEEP_OVERHEAD
906 time.sleep(sleep_time)
908 miss = t - (time.time() - start)
909 self.msg("starting %s [miss %.3f pid %d]" % (self, miss, pid))
912 print(("EXCEPTION in child PID %d, conversation %s" % (pid, self)),
914 traceback.print_exc(sys.stderr)
920 def replay(self, context=None):
923 for p in self.packets:
924 now = time.time() - start
925 gap = p.timestamp - now
926 sleep_time = gap - SLEEP_OVERHEAD
928 time.sleep(sleep_time)
930 miss = p.timestamp - (time.time() - start)
932 self.msg("packet %s [miss %.3f pid %d]" % (p, miss,
935 p.play(self, context)
937 def guess_client_server(self, server_clue=None):
938 """Have a go at deciding who is the server and who is the client.
939 returns (client, server)
941 a, b = self.endpoints
943 if self.client_balance < 0:
946 # in the absense of a clue, we will fall through to assuming
947 # the lowest number is the server (which is usually true).
949 if self.client_balance == 0 and server_clue == b:
954 def forget_packets_outside_window(self, s, e):
955 """Prune any packets outside the timne window we're interested in
957 :param s: start of the window
958 :param e: end of the window
960 self.packets = [p for p in self.packets if s <= p.timestamp <= e]
961 self.start_time = self.packets[0].timestamp if self.packets else None
963 def renormalise_times(self, start_time):
964 """Adjust the packet start times relative to the new start time."""
965 for p in self.packets:
966 p.timestamp -= start_time
968 if self.start_time is not None:
969 self.start_time -= start_time
972 class DnsHammer(Conversation):
973 """A lightweight conversation that generates a lot of dns:0 packets on
976 def __init__(self, dns_rate, duration):
977 n = int(dns_rate * duration)
978 self.times = [random.uniform(0, duration) for i in range(n)]
981 self.duration = duration
983 self.msg = random_colour_print()
986 return ("<DnsHammer %d packets over %.1fs (rate %.2f)>" %
987 (len(self.times), self.duration, self.rate))
989 def replay_in_fork_with_delay(self, start, context=None, account=None):
990 return Conversation.replay_in_fork_with_delay(self,
995 def replay(self, context=None):
997 fn = traffic_packets.packet_dns_0
999 now = time.time() - start
1001 sleep_time = gap - SLEEP_OVERHEAD
1003 time.sleep(sleep_time)
1006 miss = t - (time.time() - start)
1007 self.msg("packet %s [miss %.3f pid %d]" % (t, miss,
1011 packet_start = time.time()
1013 fn(self, self, context)
1015 duration = end - packet_start
1016 print("%f\tDNS\tdns\t0\t%f\tTrue\t" % (end, duration))
1017 except Exception as e:
1019 duration = end - packet_start
1020 print("%f\tDNS\tdns\t0\t%f\tFalse\t%s" % (end, duration, e))
1023 def ingest_summaries(files, dns_mode='count'):
1024 """Load a summary traffic summary file and generated Converations from it.
1027 dns_counts = defaultdict(int)
1030 if isinstance(f, str):
1032 print("Ingesting %s" % (f.name,), file=sys.stderr)
1034 p = Packet.from_line(line)
1035 if p.protocol == 'dns' and dns_mode != 'include':
1036 dns_counts[p.opcode] += 1
1045 start_time = min(p.timestamp for p in packets)
1046 last_packet = max(p.timestamp for p in packets)
1048 print("gathering packets into conversations", file=sys.stderr)
1049 conversations = OrderedDict()
1051 p.timestamp -= start_time
1052 c = conversations.get(p.endpoints)
1055 conversations[p.endpoints] = c
1058 # We only care about conversations with actual traffic, so we
1059 # filter out conversations with nothing to say. We do that here,
1060 # rather than earlier, because those empty packets contain useful
1061 # hints as to which end of the conversation was the client.
1062 conversation_list = []
1063 for c in conversations.values():
1065 conversation_list.append(c)
1067 # This is obviously not correct, as many conversations will appear
1068 # to start roughly simultaneously at the beginning of the snapshot.
1069 # To which we say: oh well, so be it.
1070 duration = float(last_packet - start_time)
1071 mean_interval = len(conversations) / duration
1073 return conversation_list, mean_interval, duration, dns_counts
1076 def guess_server_address(conversations):
1077 # we guess the most common address.
1078 addresses = Counter()
1079 for c in conversations:
1080 addresses.update(c.endpoints)
1082 return addresses.most_common(1)[0]
1085 def stringify_keys(x):
1087 for k, v in x.items():
1093 def unstringify_keys(x):
1095 for k, v in x.items():
1096 t = tuple(str(k).split('\t'))
1101 class TrafficModel(object):
1102 def __init__(self, n=3):
1104 self.query_details = {}
1106 self.dns_opcounts = defaultdict(int)
1107 self.cumulative_duration = 0.0
1108 self.conversation_rate = [0, 1]
1110 def learn(self, conversations, dns_opcounts={}):
1113 key = (NON_PACKET,) * (self.n - 1)
1115 server = guess_server_address(conversations)
1117 for k, v in dns_opcounts.items():
1118 self.dns_opcounts[k] += v
1120 if len(conversations) > 1:
1122 conversations[-1].start_time - conversations[0].start_time
1123 self.conversation_rate[0] = len(conversations)
1124 self.conversation_rate[1] = elapsed
1126 for c in conversations:
1127 client, server = c.guess_client_server(server)
1128 cum_duration += c.get_duration()
1129 key = (NON_PACKET,) * (self.n - 1)
1134 elapsed = p.timestamp - prev
1136 if elapsed > WAIT_THRESHOLD:
1137 # add the wait as an extra state
1138 wait = 'wait:%d' % (math.log(max(1.0,
1139 elapsed * WAIT_SCALE)))
1140 self.ngrams.setdefault(key, []).append(wait)
1141 key = key[1:] + (wait,)
1143 short_p = p.as_packet_type()
1144 self.query_details.setdefault(short_p,
1145 []).append(tuple(p.extra))
1146 self.ngrams.setdefault(key, []).append(short_p)
1147 key = key[1:] + (short_p,)
1149 self.cumulative_duration += cum_duration
1151 self.ngrams.setdefault(key, []).append(NON_PACKET)
1155 for k, v in self.ngrams.items():
1157 ngrams[k] = dict(Counter(v))
1160 for k, v in self.query_details.items():
1161 query_details[k] = dict(Counter('\t'.join(x) if x else '-'
1166 'query_details': query_details,
1167 'cumulative_duration': self.cumulative_duration,
1168 'conversation_rate': self.conversation_rate,
1170 d['dns'] = self.dns_opcounts
1172 if isinstance(f, str):
1175 json.dump(d, f, indent=2)
1178 if isinstance(f, str):
1183 for k, v in d['ngrams'].items():
1184 k = tuple(str(k).split('\t'))
1185 values = self.ngrams.setdefault(k, [])
1186 for p, count in v.items():
1187 values.extend([str(p)] * count)
1189 for k, v in d['query_details'].items():
1190 values = self.query_details.setdefault(str(k), [])
1191 for p, count in v.items():
1193 values.extend([()] * count)
1195 values.extend([tuple(str(p).split('\t'))] * count)
1198 for k, v in d['dns'].items():
1199 self.dns_opcounts[k] += v
1201 self.cumulative_duration = d['cumulative_duration']
1202 self.conversation_rate = d['conversation_rate']
1204 def construct_conversation(self, timestamp=0.0, client=2, server=1,
1205 hard_stop=None, packet_rate=1):
1206 """Construct a individual converation from the model."""
1208 c = Conversation(timestamp, (server, client))
1210 key = (NON_PACKET,) * (self.n - 1)
1212 while key in self.ngrams:
1213 p = random.choice(self.ngrams.get(key, NON_PACKET))
1216 if p in self.query_details:
1217 extra = random.choice(self.query_details[p])
1221 protocol, opcode = p.split(':', 1)
1222 if protocol == 'wait':
1223 log_wait_time = int(opcode) + random.random()
1224 wait = math.exp(log_wait_time) / (WAIT_SCALE * packet_rate)
1227 log_wait = random.uniform(*NO_WAIT_LOG_TIME_RANGE)
1228 wait = math.exp(log_wait) / packet_rate
1230 if hard_stop is not None and timestamp > hard_stop:
1232 c.add_short_packet(timestamp, protocol, opcode, extra)
1234 key = key[1:] + (p,)
1238 def generate_conversations(self, rate, duration, packet_rate=1):
1239 """Generate a list of conversations from the model."""
1241 # We run the simulation for at least ten times as long as our
1242 # desired duration, and take a section near the start.
1243 rate_n, rate_t = self.conversation_rate
1245 duration2 = max(rate_t, duration * 2)
1246 n = rate * duration2 * rate_n / rate_t
1253 start = end - duration
1255 while client < n + 2:
1256 start = random.uniform(0, duration2)
1257 c = self.construct_conversation(start,
1260 hard_stop=(duration2 * 5),
1261 packet_rate=packet_rate)
1263 c.forget_packets_outside_window(start, end)
1264 c.renormalise_times(start)
1266 conversations.append(c)
1269 print(("we have %d conversations at rate %f" %
1270 (len(conversations), rate)), file=sys.stderr)
1271 conversations.sort()
1272 return conversations
1277 'rpc_netlogon': '06',
1278 'kerberos': '06', # ratio 16248:258
1289 'smb_netlogon': '11',
1295 ('browser', '0x01'): 'Host Announcement (0x01)',
1296 ('browser', '0x02'): 'Request Announcement (0x02)',
1297 ('browser', '0x08'): 'Browser Election Request (0x08)',
1298 ('browser', '0x09'): 'Get Backup List Request (0x09)',
1299 ('browser', '0x0c'): 'Domain/Workgroup Announcement (0x0c)',
1300 ('browser', '0x0f'): 'Local Master Announcement (0x0f)',
1301 ('cldap', '3'): 'searchRequest',
1302 ('cldap', '5'): 'searchResDone',
1303 ('dcerpc', '0'): 'Request',
1304 ('dcerpc', '11'): 'Bind',
1305 ('dcerpc', '12'): 'Bind_ack',
1306 ('dcerpc', '13'): 'Bind_nak',
1307 ('dcerpc', '14'): 'Alter_context',
1308 ('dcerpc', '15'): 'Alter_context_resp',
1309 ('dcerpc', '16'): 'AUTH3',
1310 ('dcerpc', '2'): 'Response',
1311 ('dns', '0'): 'query',
1312 ('dns', '1'): 'response',
1313 ('drsuapi', '0'): 'DsBind',
1314 ('drsuapi', '12'): 'DsCrackNames',
1315 ('drsuapi', '13'): 'DsWriteAccountSpn',
1316 ('drsuapi', '1'): 'DsUnbind',
1317 ('drsuapi', '2'): 'DsReplicaSync',
1318 ('drsuapi', '3'): 'DsGetNCChanges',
1319 ('drsuapi', '4'): 'DsReplicaUpdateRefs',
1320 ('epm', '3'): 'Map',
1321 ('kerberos', ''): '',
1322 ('ldap', '0'): 'bindRequest',
1323 ('ldap', '1'): 'bindResponse',
1324 ('ldap', '2'): 'unbindRequest',
1325 ('ldap', '3'): 'searchRequest',
1326 ('ldap', '4'): 'searchResEntry',
1327 ('ldap', '5'): 'searchResDone',
1328 ('ldap', ''): '*** Unknown ***',
1329 ('lsarpc', '14'): 'lsa_LookupNames',
1330 ('lsarpc', '15'): 'lsa_LookupSids',
1331 ('lsarpc', '39'): 'lsa_QueryTrustedDomainInfoBySid',
1332 ('lsarpc', '40'): 'lsa_SetTrustedDomainInfo',
1333 ('lsarpc', '6'): 'lsa_OpenPolicy',
1334 ('lsarpc', '76'): 'lsa_LookupSids3',
1335 ('lsarpc', '77'): 'lsa_LookupNames4',
1336 ('nbns', '0'): 'query',
1337 ('nbns', '1'): 'response',
1338 ('rpc_netlogon', '21'): 'NetrLogonDummyRoutine1',
1339 ('rpc_netlogon', '26'): 'NetrServerAuthenticate3',
1340 ('rpc_netlogon', '29'): 'NetrLogonGetDomainInfo',
1341 ('rpc_netlogon', '30'): 'NetrServerPasswordSet2',
1342 ('rpc_netlogon', '39'): 'NetrLogonSamLogonEx',
1343 ('rpc_netlogon', '40'): 'DsrEnumerateDomainTrusts',
1344 ('rpc_netlogon', '45'): 'NetrLogonSamLogonWithFlags',
1345 ('rpc_netlogon', '4'): 'NetrServerReqChallenge',
1346 ('samr', '0',): 'Connect',
1347 ('samr', '16'): 'GetAliasMembership',
1348 ('samr', '17'): 'LookupNames',
1349 ('samr', '18'): 'LookupRids',
1350 ('samr', '19'): 'OpenGroup',
1351 ('samr', '1'): 'Close',
1352 ('samr', '25'): 'QueryGroupMember',
1353 ('samr', '34'): 'OpenUser',
1354 ('samr', '36'): 'QueryUserInfo',
1355 ('samr', '39'): 'GetGroupsForUser',
1356 ('samr', '3'): 'QuerySecurity',
1357 ('samr', '5'): 'LookupDomain',
1358 ('samr', '64'): 'Connect5',
1359 ('samr', '6'): 'EnumDomains',
1360 ('samr', '7'): 'OpenDomain',
1361 ('samr', '8'): 'QueryDomainInfo',
1362 ('smb', '0x04'): 'Close (0x04)',
1363 ('smb', '0x24'): 'Locking AndX (0x24)',
1364 ('smb', '0x2e'): 'Read AndX (0x2e)',
1365 ('smb', '0x32'): 'Trans2 (0x32)',
1366 ('smb', '0x71'): 'Tree Disconnect (0x71)',
1367 ('smb', '0x72'): 'Negotiate Protocol (0x72)',
1368 ('smb', '0x73'): 'Session Setup AndX (0x73)',
1369 ('smb', '0x74'): 'Logoff AndX (0x74)',
1370 ('smb', '0x75'): 'Tree Connect AndX (0x75)',
1371 ('smb', '0xa2'): 'NT Create AndX (0xa2)',
1372 ('smb2', '0'): 'NegotiateProtocol',
1373 ('smb2', '11'): 'Ioctl',
1374 ('smb2', '14'): 'Find',
1375 ('smb2', '16'): 'GetInfo',
1376 ('smb2', '18'): 'Break',
1377 ('smb2', '1'): 'SessionSetup',
1378 ('smb2', '2'): 'SessionLogoff',
1379 ('smb2', '3'): 'TreeConnect',
1380 ('smb2', '4'): 'TreeDisconnect',
1381 ('smb2', '5'): 'Create',
1382 ('smb2', '6'): 'Close',
1383 ('smb2', '8'): 'Read',
1384 ('smb_netlogon', '0x12'): 'SAM LOGON request from client (0x12)',
1385 ('smb_netlogon', '0x17'): ('SAM Active Directory Response - '
1386 'user unknown (0x17)'),
1387 ('srvsvc', '16'): 'NetShareGetInfo',
1388 ('srvsvc', '21'): 'NetSrvGetInfo',
1392 def expand_short_packet(p, timestamp, src, dest, extra):
1393 protocol, opcode = p.split(':', 1)
1394 desc = OP_DESCRIPTIONS.get((protocol, opcode), '')
1395 ip_protocol = IP_PROTOCOLS.get(protocol, '06')
1397 line = [timestamp, ip_protocol, '', src, dest, protocol, opcode, desc]
1399 return '\t'.join(line)
1402 def replay(conversations,
1411 context = ReplayContext(server=host,
1416 if len(accounts) < len(conversations):
1417 print(("we have %d accounts but %d conversations" %
1418 (accounts, conversations)), file=sys.stderr)
1421 sorted(conversations, key=lambda x: x.start_time, reverse=True),
1424 # Set the process group so that the calling scripts are not killed
1425 # when the forked child processes are killed.
1430 if duration is None:
1431 # end 1 second after the last packet of the last conversation
1432 # to start. Conversations other than the last could still be
1433 # going, but we don't care.
1434 duration = cstack[0][0].packets[-1].timestamp + 1.0
1435 print("We will stop after %.1f seconds" % duration,
1438 end = start + duration
1440 print("Replaying traffic for %u conversations over %d seconds"
1441 % (len(conversations), duration))
1445 dns_hammer = DnsHammer(dns_rate, duration)
1446 cstack.append((dns_hammer, None))
1450 # we spawn a batch, wait for finishers, then spawn another
1452 batch_end = min(now + 2.0, end)
1456 c, account = cstack.pop()
1457 if c.start_time + start > batch_end:
1458 cstack.append((c, account))
1462 pid = c.replay_in_fork_with_delay(start, context, account)
1466 fork_time += elapsed
1468 print("forked %s in pid %s (in %fs)" % (c, pid,
1473 print(("forked %d times in %f seconds (avg %f)" %
1474 (fork_n, fork_time, fork_time / fork_n)),
1477 debug(2, "no forks in batch ending %f" % batch_end)
1479 while time.time() < batch_end - 1.0:
1482 pid, status = os.waitpid(-1, os.WNOHANG)
1483 except OSError as e:
1484 if e.errno != 10: # no child processes
1488 c = children.pop(pid, None)
1489 print(("process %d finished conversation %s;"
1491 (pid, c, len(children))), file=sys.stderr)
1493 if time.time() >= end:
1494 print("time to stop", file=sys.stderr)
1498 print("EXCEPTION in parent", file=sys.stderr)
1499 traceback.print_exc()
1501 for s in (15, 15, 9):
1502 print(("killing %d children with -%d" %
1503 (len(children), s)), file=sys.stderr)
1504 for pid in children:
1507 except OSError as e:
1508 if e.errno != 3: # don't fail if it has already died
1511 end = time.time() + 1
1514 pid, status = os.waitpid(-1, os.WNOHANG)
1515 except OSError as e:
1519 c = children.pop(pid, None)
1520 print(("kill -%d %d KILLED conversation %s; "
1522 (s, pid, c, len(children))),
1524 if time.time() >= end:
1532 print("%d children are missing" % len(children),
1535 # there may be stragglers that were forked just as ^C was hit
1536 # and don't appear in the list of children. We can get them
1537 # with killpg, but that will also kill us, so this is^H^H would be
1538 # goodbye, except we cheat and pretend to use ^C (SIG_INTERRUPT),
1539 # so as not to have to fuss around writing signal handlers.
1542 except KeyboardInterrupt:
1543 print("ignoring fake ^C", file=sys.stderr)
1546 def openLdb(host, creds, lp):
1547 session = system_session()
1548 ldb = SamDB(url="ldap://%s" % host,
1549 session_info=session,
1555 def ou_name(ldb, instance_id):
1556 """Generate an ou name from the instance id"""
1557 return "ou=instance-%d,ou=traffic_replay,%s" % (instance_id,
1561 def create_ou(ldb, instance_id):
1562 """Create an ou, all created user and machine accounts will belong to it.
1564 This allows all the created resources to be cleaned up easily.
1566 ou = ou_name(ldb, instance_id)
1568 ldb.add({"dn": ou.split(',', 1)[1],
1569 "objectclass": "organizationalunit"})
1570 except LdbError as e:
1571 (status, _) = e.args
1572 # ignore already exists
1577 "objectclass": "organizationalunit"})
1578 except LdbError as e:
1579 (status, _) = e.args
1580 # ignore already exists
1586 class ConversationAccounts(object):
1587 """Details of the machine and user accounts associated with a conversation.
1589 def __init__(self, netbios_name, machinepass, username, userpass):
1590 self.netbios_name = netbios_name
1591 self.machinepass = machinepass
1592 self.username = username
1593 self.userpass = userpass
1596 def generate_replay_accounts(ldb, instance_id, number, password):
1597 """Generate a series of unique machine and user account names."""
1599 generate_traffic_accounts(ldb, instance_id, number, password)
1601 for i in range(1, number + 1):
1602 netbios_name = "STGM-%d-%d" % (instance_id, i)
1603 username = "STGU-%d-%d" % (instance_id, i)
1605 account = ConversationAccounts(netbios_name, password, username,
1607 accounts.append(account)
1611 def generate_traffic_accounts(ldb, instance_id, number, password):
1612 """Create the specified number of user and machine accounts.
1614 As accounts are not explicitly deleted between runs. This function starts
1615 with the last account and iterates backwards stopping either when it
1616 finds an already existing account or it has generated all the required
1619 print(("Generating machine and conversation accounts, "
1620 "as required for %d conversations" % number),
1623 for i in range(number, 0, -1):
1625 netbios_name = "STGM-%d-%d" % (instance_id, i)
1626 create_machine_account(ldb, instance_id, netbios_name, password)
1628 except LdbError as e:
1629 (status, _) = e.args
1635 print("Added %d new machine accounts" % added,
1639 for i in range(number, 0, -1):
1641 username = "STGU-%d-%d" % (instance_id, i)
1642 create_user_account(ldb, instance_id, username, password)
1644 except LdbError as e:
1645 (status, _) = e.args
1652 print("Added %d new user accounts" % added,
1656 def create_machine_account(ldb, instance_id, netbios_name, machinepass):
1657 """Create a machine account via ldap."""
1659 ou = ou_name(ldb, instance_id)
1660 dn = "cn=%s,%s" % (netbios_name, ou)
1662 '"' + machinepass.encode('utf-8') + '"', 'utf-8'
1663 ).encode('utf-16-le')
1667 "objectclass": "computer",
1668 "sAMAccountName": "%s$" % netbios_name,
1669 "userAccountControl":
1670 str(UF_TRUSTED_FOR_DELEGATION | UF_SERVER_TRUST_ACCOUNT),
1671 "unicodePwd": utf16pw})
1673 duration = end - start
1674 print("%f\t0\tcreate\tmachine\t%f\tTrue\t" % (end, duration))
1677 def create_user_account(ldb, instance_id, username, userpass):
1678 """Create a user account via ldap."""
1679 ou = ou_name(ldb, instance_id)
1680 user_dn = "cn=%s,%s" % (username, ou)
1682 '"' + userpass.encode('utf-8') + '"', 'utf-8'
1683 ).encode('utf-16-le')
1687 "objectclass": "user",
1688 "sAMAccountName": username,
1689 "userAccountControl": str(UF_NORMAL_ACCOUNT),
1690 "unicodePwd": utf16pw
1693 # grant user write permission to do things like write account SPN
1694 sdutils = sd_utils.SDUtils(ldb)
1695 sdutils.dacl_add_ace(user_dn, "(A;;WP;;;PS)")
1698 duration = end - start
1699 print("%f\t0\tcreate\tuser\t%f\tTrue\t" % (end, duration))
1702 def create_group(ldb, instance_id, name):
1703 """Create a group via ldap."""
1705 ou = ou_name(ldb, instance_id)
1706 dn = "cn=%s,%s" % (name, ou)
1710 "objectclass": "group",
1711 "sAMAccountName": name,
1714 duration = end - start
1715 print("%f\t0\tcreate\tgroup\t%f\tTrue\t" % (end, duration))
1718 def user_name(instance_id, i):
1719 """Generate a user name based in the instance id"""
1720 return "STGU-%d-%d" % (instance_id, i)
1723 def generate_users(ldb, instance_id, number, password):
1724 """Add users to the server"""
1726 for i in range(number, 0, -1):
1728 username = user_name(instance_id, i)
1729 create_user_account(ldb, instance_id, username, password)
1731 except LdbError as e:
1732 (status, _) = e.args
1733 # Stop if entry exists
1742 def group_name(instance_id, i):
1743 """Generate a group name from instance id."""
1744 return "STGG-%d-%d" % (instance_id, i)
1747 def generate_groups(ldb, instance_id, number):
1748 """Create the required number of groups on the server."""
1750 for i in range(number, 0, -1):
1752 name = group_name(instance_id, i)
1753 create_group(ldb, instance_id, name)
1755 except LdbError as e:
1756 (status, _) = e.args
1757 # Stop if entry exists
1765 def clean_up_accounts(ldb, instance_id):
1766 """Remove the created accounts and groups from the server."""
1767 ou = ou_name(ldb, instance_id)
1769 ldb.delete(ou, ["tree_delete:1"])
1770 except LdbError as e:
1771 (status, _) = e.args
1772 # ignore does not exist
1777 def generate_users_and_groups(ldb, instance_id, password,
1778 number_of_users, number_of_groups,
1780 """Generate the required users and groups, allocating the users to
1785 create_ou(ldb, instance_id)
1787 print("Generating dummy user accounts", file=sys.stderr)
1788 users_added = generate_users(ldb, instance_id, number_of_users, password)
1790 if number_of_groups > 0:
1791 print("Generating dummy groups", file=sys.stderr)
1792 groups_added = generate_groups(ldb, instance_id, number_of_groups)
1794 if group_memberships > 0:
1795 print("Assigning users to groups", file=sys.stderr)
1796 assignments = assign_groups(number_of_groups,
1801 print("Adding users to groups", file=sys.stderr)
1802 add_users_to_groups(ldb, instance_id, assignments)
1804 if (groups_added > 0 and users_added == 0 and
1805 number_of_groups != groups_added):
1806 print("Warning: the added groups will contain no members",
1809 print(("Added %d users, %d groups and %d group memberships" %
1810 (users_added, groups_added, len(assignments))),
1814 def assign_groups(number_of_groups,
1819 """Allocate users to groups.
1821 The intention is to have a few users that belong to most groups, while
1822 the majority of users belong to a few groups.
1824 A few groups will contain most users, with the remaining only having a
1828 def generate_user_distribution(n):
1829 """Probability distribution of a user belonging to a group.
1832 for x in range(1, n + 1):
1837 def generate_group_distribution(n):
1838 """Probability distribution of a group containing a user."""
1840 for x in range(1, n + 1):
1846 if group_memberships <= 0:
1849 group_dist = generate_group_distribution(number_of_groups)
1850 user_dist = generate_user_distribution(number_of_users)
1852 # Calculate the number of group menberships required
1853 group_memberships = math.ceil(
1854 float(group_memberships) *
1855 (float(users_added) / float(number_of_users)))
1857 existing_users = number_of_users - users_added - 1
1858 existing_groups = number_of_groups - groups_added - 1
1859 while len(assignments) < group_memberships:
1860 user = random.randint(0, number_of_users - 1)
1861 group = random.randint(0, number_of_groups - 1)
1862 probability = group_dist[group] * user_dist[user]
1864 if ((random.random() < probability * 10000) and
1865 (group > existing_groups or user > existing_users)):
1866 # the + 1 converts the array index to the corresponding
1867 # group or user number
1868 assignments.add(((user + 1), (group + 1)))
1873 def add_users_to_groups(db, instance_id, assignments):
1874 """Add users to their assigned groups.
1876 Takes the list of (group,user) tuples generated by assign_groups and
1877 assign the users to their specified groups."""
1879 ou = ou_name(db, instance_id)
1882 return("cn=%s,%s" % (name, ou))
1884 for (user, group) in assignments:
1885 user_dn = build_dn(user_name(instance_id, user))
1886 group_dn = build_dn(group_name(instance_id, group))
1889 m.dn = ldb.Dn(db, group_dn)
1890 m["member"] = ldb.MessageElement(user_dn, ldb.FLAG_MOD_ADD, "member")
1894 duration = end - start
1895 print("%f\t0\tadd\tuser\t%f\tTrue\t" % (end, duration))
1898 def generate_stats(statsdir, timing_file):
1899 """Generate and print the summary stats for a run."""
1900 first = sys.float_info.max
1906 unique_converations = set()
1909 if timing_file is not None:
1910 tw = timing_file.write
1915 tw("time\tconv\tprotocol\ttype\tduration\tsuccessful\terror\n")
1917 for filename in os.listdir(statsdir):
1918 path = os.path.join(statsdir, filename)
1919 with open(path, 'r') as f:
1922 fields = line.rstrip('\n').split('\t')
1923 conversation = fields[1]
1924 protocol = fields[2]
1925 packet_type = fields[3]
1926 latency = float(fields[4])
1927 first = min(float(fields[0]) - latency, first)
1928 last = max(float(fields[0]), last)
1930 if protocol not in latencies:
1931 latencies[protocol] = {}
1932 if packet_type not in latencies[protocol]:
1933 latencies[protocol][packet_type] = []
1935 latencies[protocol][packet_type].append(latency)
1937 if protocol not in failures:
1938 failures[protocol] = {}
1939 if packet_type not in failures[protocol]:
1940 failures[protocol][packet_type] = 0
1942 if fields[5] == 'True':
1946 failures[protocol][packet_type] += 1
1948 if conversation not in unique_converations:
1949 unique_converations.add(conversation)
1953 except (ValueError, IndexError):
1954 # not a valid line print and ignore
1955 print(line, file=sys.stderr)
1957 duration = last - first
1961 success_rate = successful / duration
1965 failure_rate = failed / duration
1967 print("Total conversations: %10d" % conversations)
1968 print("Successful operations: %10d (%.3f per second)"
1969 % (successful, success_rate))
1970 print("Failed operations: %10d (%.3f per second)"
1971 % (failed, failure_rate))
1973 print("Protocol Op Code Description "
1974 " Count Failed Mean Median "
1977 protocols = sorted(latencies.keys())
1978 for protocol in protocols:
1979 packet_types = sorted(latencies[protocol], key=opcode_key)
1980 for packet_type in packet_types:
1981 values = latencies[protocol][packet_type]
1982 values = sorted(values)
1984 failed = failures[protocol][packet_type]
1985 mean = sum(values) / count
1986 median = calc_percentile(values, 0.50)
1987 percentile = calc_percentile(values, 0.95)
1988 rng = values[-1] - values[0]
1990 desc = OP_DESCRIPTIONS.get((protocol, packet_type), '')
1991 if sys.stdout.isatty:
1992 print("%-12s %4s %-35s %12d %12d %12.6f "
1993 "%12.6f %12.6f %12.6f %12.6f"
2005 print("%s\t%s\t%s\t%d\t%d\t%f\t%f\t%f\t%f\t%f"
2019 """Sort key for the operation code to ensure that it sorts numerically"""
2021 return "%03d" % int(v)
2026 def calc_percentile(values, percentile):
2027 """Calculate the specified percentile from the list of values.
2029 Assumes the list is sorted in ascending order.
2034 k = (len(values) - 1) * percentile
2038 return values[int(k)]
2039 d0 = values[int(f)] * (c - k)
2040 d1 = values[int(c)] * (k - f)
2044 def mk_masked_dir(*path):
2045 """In a testenv we end up with 0777 diectories that look an alarming
2046 green colour with ls. Use umask to avoid that."""
2047 d = os.path.join(*path)
2048 mask = os.umask(0o077)