from samba.logger import get_samba_logger
import bisect
+CURRENT_MODEL_VERSION = 2 # save as this
+REQUIRED_MODEL_VERSION = 2 # load accepts this or greater
SLEEP_OVERHEAD = 3e-4
# we don't use None, because it complicates [de]serialisation
self.n = n
self.dns_opcounts = defaultdict(int)
self.cumulative_duration = 0.0
- self.conversation_rate = [0, 1]
+ self.packet_rate = [0, 1]
def learn(self, conversations, dns_opcounts={}):
prev = 0.0
self.dns_opcounts[k] += v
if len(conversations) > 1:
- elapsed =\
- conversations[-1].start_time - conversations[0].start_time
- self.conversation_rate[0] = len(conversations)
- self.conversation_rate[1] = elapsed
+ first = conversations[0].start_time
+ total = 0
+ last = first + 0.1
+ for c in conversations:
+ total += len(c)
+ last = max(last, c.packets[-1].timestamp)
+
+ self.packet_rate[0] = total
+ self.packet_rate[1] = last - first
for c in conversations:
client, server = c.guess_client_server(server)
'ngrams': ngrams,
'query_details': query_details,
'cumulative_duration': self.cumulative_duration,
- 'conversation_rate': self.conversation_rate,
+ 'packet_rate': self.packet_rate,
+ 'version': CURRENT_MODEL_VERSION
}
d['dns'] = self.dns_opcounts
d = json.load(f)
+ try:
+ version = d["version"]
+ if version < REQUIRED_MODEL_VERSION:
+ raise ValueError("the model file is version %d; "
+ "version %d is required" %
+ (version, REQUIRED_MODEL_VERSION))
+ except KeyError:
+ raise ValueError("the model file lacks a version number; "
+ "version %d is required" %
+ (REQUIRED_MODEL_VERSION))
+
for k, v in d['ngrams'].items():
k = tuple(str(k).split('\t'))
values = self.ngrams.setdefault(k, [])
self.dns_opcounts[k] += v
self.cumulative_duration = d['cumulative_duration']
- self.conversation_rate = d['conversation_rate']
-
- def construct_conversation(self, timestamp=0.0, client=2, server=1,
- hard_stop=None, replay_speed=1):
- """Construct a individual converation from the model."""
-
- c = Conversation(timestamp, (server, client), conversation_id=client)
-
+ self.packet_rate = d['packet_rate']
+
+ def construct_conversation_sequence(self, timestamp=0.0,
+ hard_stop=None,
+ replay_speed=1,
+ ignore_before=0):
+ """Construct an individual conversation packet sequence from the
+ model.
+ """
+ c = []
key = (NON_PACKET,) * (self.n - 1)
+ if ignore_before is None:
+ ignore_before = timestamp - 1
- while key in self.ngrams:
- p = random.choice(self.ngrams.get(key, NON_PACKET))
+ while True:
+ p = random.choice(self.ngrams.get(key, (NON_PACKET,)))
if p == NON_PACKET:
break
timestamp += wait
if hard_stop is not None and timestamp > hard_stop:
break
- c.add_short_packet(timestamp, protocol, opcode, extra)
+ if timestamp >= ignore_before:
+ c.append((timestamp, protocol, opcode, extra))
key = key[1:] + (p,)
return c
- def generate_conversations(self, rate, duration, replay_speed=1):
+ def generate_conversations(self, scale, duration, replay_speed=1,
+ server=1, client=2):
"""Generate a list of conversations from the model."""
- # We run the simulation for at least ten times as long as our
- # desired duration, and take a section near the start.
- rate_n, rate_t = self.conversation_rate
+ # We run the simulation for ten times as long as our desired
+ # duration, and take the section at the end.
+ lead_in = 9 * duration
+ rate_n, rate_t = self.packet_rate
+ target_packets = int(duration * scale * rate_n / rate_t)
- duration2 = max(rate_t, duration * 2)
- n = rate * duration2 * rate_n / rate_t
+ conversations = []
+ n_packets = 0
+
+ while n_packets < target_packets:
+ start = random.uniform(-lead_in, duration)
+ c = self.construct_conversation_sequence(start,
+ hard_stop=duration,
+ replay_speed=replay_speed,
+ ignore_before=0)
+ conversations.append(c)
+ n_packets += len(c)
+
+ print(("we have %d packets (target %d) in %d conversations at scale %f"
+ % (n_packets, target_packets, len(conversations), scale)),
+ file=sys.stderr)
+ conversations.sort() # sorts by first element == start time
+ return seq_to_conversations(conversations)
- server = 1
- client = 2
- conversations = []
- end = duration2
- start = end - duration
-
- while client < n + 2:
- start = random.uniform(0, duration2)
- c = self.construct_conversation(start,
- client,
- server,
- hard_stop=(duration2 * 5),
- replay_speed=replay_speed)
-
- c.forget_packets_outside_window(start, end)
- c.renormalise_times(start)
- if len(c) != 0:
- conversations.append(c)
+def seq_to_conversations(seq, server=1, client=2):
+ conversations = []
+ for s in seq:
+ if s:
+ c = Conversation(s[0][0], (server, client), s)
client += 1
-
- print(("we have %d conversations at rate %f" %
- (len(conversations), rate)), file=sys.stderr)
- conversations.sort()
- return conversations
+ conversations.append(c)
+ return conversations
IP_PROTOCOLS = {
"-": 1
}
},
- "conversation_rate": [
- 2,
- 0.12712717056274414
- ],
"dns": {
"1": 9,
"0": 9
},
+ "packet_rate": [
+ 50,
+ 0.32482
+ ],
"query_details": {
"rpc_netlogon:29": {
"-": 1
"": 1
}
},
- "cumulative_duration": 0.39243292808532715
+ "cumulative_duration": 0.39243292808532715,
+ "version": 2
}
\ No newline at end of file
{
+ "packet_rate": [
+ 10,
+ 0.22707200050354004
+ ],
+ "query_details": {
+ "rpc_netlogon:29": {
+ "-": 1
+ },
+ "cldap:3": {
+ "\t\t\tNetlogon\t\t\t": 3
+ },
+ "ldap:3": {
+ "\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1,
+ "2\tDC,DC\t\tcn\t\t\t": 1
+ },
+ "ldap:2": {
+ "\t\t\t\t\t\t": 1
+ },
+ "kerberos:": {
+ "": 1
+ }
+ },
"ngrams": {
"-\t-": {
"cldap:3": 1,
"-": 1
}
},
- "conversation_rate": [
- 2,
- 0.12712717056274414
- ],
+ "version": 2,
"dns": {
"1": 9,
"0": 9
},
- "query_details": {
- "rpc_netlogon:29": {
- "-": 1
- },
- "cldap:3": {
- "\t\t\tNetlogon\t\t\t": 3
- },
- "ldap:3": {
- "\t\t\tsubschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities\t\t\t": 1,
- "2\tDC,DC\t\tcn\t\t\t": 1
- },
- "ldap:2": {
- "\t\t\t\t\t\t": 1
- },
- "kerberos:": {
- "": 1
- }
- },
"cumulative_duration": 0.39243292808532715
}
\ No newline at end of file
--- /dev/null
+0.011388 06 2 1 ldap 3 searchRequest 2 DC,DC cn
+0.221447 06 2 1 ldap 2 unbindRequest
+0.460878 06 3 1 ldap 3 searchRequest 2 DC,DC cn
+0.581933 11 4 1 cldap 3 searchRequest Netlogon
+0.596977 11 4 1 cldap 3 searchRequest Netlogon
+0.611184 11 4 1 cldap 3 searchRequest Netlogon
+0.666808 06 3 1 ldap 2 unbindRequest
+0.744297 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.768994 06 4 1 kerberos
+0.772476 06 4 1 ldap 3 searchRequest 2 DC,DC cn
+0.805442 11 5 1 cldap 3 searchRequest Netlogon
+0.805536 11 5 1 cldap 3 searchRequest Netlogon
+0.807659 11 5 1 cldap 3 searchRequest Netlogon
+0.808614 11 5 1 cldap 3 searchRequest Netlogon
+0.808819 11 5 1 cldap 3 searchRequest Netlogon
+0.865384 06 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
+0.973595 06 5 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.974012 06 5 1 kerberos
--- /dev/null
+0.011519 11 2 1 cldap 3 searchRequest Netlogon
+0.012916 11 2 1 cldap 3 searchRequest Netlogon
+0.158388 06 3 1 ldap 3 searchRequest 2 DC,DC cn
+0.164506 06 2 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.166151 06 2 1 kerberos
+0.166301 06 2 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
+0.258932 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.259908 06 4 1 kerberos
+0.260073 06 4 1 ldap 3 searchRequest 2 DC,DC cn
+0.286044 06 5 1 ldap 3 searchRequest 2 DC,DC cn
+0.295757 06 3 1 ldap 2 unbindRequest
+0.459791 06 5 1 ldap 2 unbindRequest
+0.553887 06 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
+0.641127 11 7 1 cldap 3 searchRequest Netlogon
+0.641297 11 7 1 cldap 3 searchRequest Netlogon
+0.783989 06 6 1 ldap 2 unbindRequest
+0.901096 06 7 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.915260 06 7 1 kerberos
+0.915711 06 7 1 ldap 3 searchRequest 2 DC,DC cn
-0.000000 06 3 5 1 ldap 3 searchRequest 2 DC,DC cn
-0.127127 11 6 1 cldap 3 searchRequest Netlogon
-0.127391 11 6 1 cldap 3 searchRequest Netlogon
-0.127689 11 6 1 cldap 3 searchRequest Netlogon
-0.227072 06 3 5 1 ldap 2 unbindRequest
-0.282995 06 16 6 1 rpc_netlogon 29 NetrLogonGetDomainInfo
-0.287553 06 18 6 1 kerberos
-0.287759 06 16 1 6 rpc_netlogon 29 NetrLogonGetDomainInfo
-0.291161 06 19 6 1 ldap 3 searchRequest subschemaSubentry,dsServiceName,namingContexts,defaultNamingContext,schemaNamingContext,configurationNamingContext,rootDomainNamingContext,supportedControl,supportedLDAPVersion,supportedLDAPPolicies,supportedSASLMechanisms,dnsHostName,ldapServiceName,serverName,supportedCapabilities
-0.292488 06 18 1 6 kerberos
+0.040433 06 2 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.059203 06 2 1 kerberos
+0.061641 06 2 1 ldap 3 searchRequest 2 DC,DC cn
+0.535074 11 3 1 cldap 3 searchRequest Netlogon
+0.535369 11 3 1 cldap 3 searchRequest Netlogon
+0.536671 11 3 1 cldap 3 searchRequest Netlogon
+0.537238 11 3 1 cldap 3 searchRequest Netlogon
+0.537362 11 3 1 cldap 3 searchRequest Netlogon
+0.602824 11 4 1 cldap 3 searchRequest Netlogon
+0.640115 11 4 1 cldap 3 searchRequest Netlogon
+0.714546 06 3 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.715865 06 3 1 kerberos
+0.716613 06 3 1 ldap 3 searchRequest 2 DC,DC cn
+0.767674 06 4 1 rpc_netlogon 29 NetrLogonGetDomainInfo
+0.778022 06 5 1 ldap 3 searchRequest 2 DC,DC cn
+0.792356 06 4 1 kerberos
+0.792763 06 4 1 ldap 3 searchRequest 2 DC,DC cn
+0.960412 06 5 1 ldap 2 unbindRequest
expected_details = {k: sorted(v) for k, v in expected.query_details.items()}
self.assertEquals(expected_details, actual_details)
self.assertEquals(expected.cumulative_duration, actual.cumulative_duration)
- self.assertEquals(expected.conversation_rate, actual.conversation_rate)
+ self.assertEquals(expected.packet_rate, actual.packet_rate)
DATA_DIR = "python/samba/tests/blackbox/testdata"
SCRIPT = "script/traffic_replay"
-FIXED = "--fixed-password trafficreplay01%"
+FIXED = "--fixed-password=trafficreplay01%"
SERVER = os.environ["SERVER"]
PASSWORD = os.environ["PASSWORD"]
USER = os.environ["USERNAME"]
-STD_OPTIONS = "-U%s%%%s %s" % (USER, PASSWORD, SERVER)
-EXPECTED_NAME = os.path.join(DATA_DIR, "traffic_replay.expected")
+CREDS = "-U%s%%%s" % (USER, PASSWORD)
+MODEL = os.path.join(DATA_DIR, "traffic-sample-very-short.model")
+EXPECTED_OUTPUT = os.path.join(DATA_DIR, "traffic_replay-%s.expected")
@contextmanager
def tearDown(self):
options = "--clean-up"
- command = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
+ command = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
self.check_run(command)
def test_generate_users_only(self):
"""
options = ("--generate-users-only --number-of-users 20 "
"--number-of-groups 5 --average-groups-per-user 2")
- command = "%s %s %s %s" % (
- SCRIPT, options, FIXED, STD_OPTIONS)
+ command = "%s %s %s %s %s" % (
+ SCRIPT, options, FIXED, CREDS, SERVER)
self.check_run(command)
+ command = "%s %s %s %s %s %s" % (
+ SCRIPT, MODEL, options, FIXED, CREDS, SERVER)
+ self.check_run(command)
+
+ def test_summary_generation(self):
+ """Ensure a summary file is generated and the contents are correct"""
+
+ for i, opts in enumerate((["--random-seed=3"],
+ ["--random-seed=4"],
+ ["--random-seed=3",
+ "--conversation-persistence=0.5"],
+ ["--random-seed=3",
+ "--old-scale",
+ "--conversation-persistence=0.95"],
+ )):
+ with temp_file(self.tempdir) as output:
+ command = ([SCRIPT, MODEL,
+ "--traffic-summary", output,
+ "-D1", "-S0.1"] +
+ opts +
+ [FIXED, CREDS, SERVER])
+ self.check_run(command)
+ expected = open(EXPECTED_OUTPUT % i).read()
+ actual = open(output).read()
+ self.assertStringsEqual(expected, actual)
+
+ def test_summary_replay_no_fixed(self):
+ """Ensure a summary file with no fixed password fails
+ """
+ command = [SCRIPT, MODEL, CREDS, SERVER]
+ self.check_exit_code(command, 1)
def test_model_replay(self):
"""Ensure a model can be replayed against a DC
"""
-
- model = "testdata/traffic-sample-very-short.model"
- command = "%s %s-D 5 %s %s" % (SCRIPT, model, FIXED, STD_OPTIONS)
+ command = [SCRIPT, MODEL,
+ FIXED,
+ '-D2', '-S0.1',
+ CREDS, SERVER]
self.check_run(command)
def test_generate_users_only_no_password(self):
"""
options = ("--generate-users-only --number-of-users 20 "
"--number-of-groups 5 --average-groups-per-user 2")
- command = "%s %s %s" % (SCRIPT, options, STD_OPTIONS)
+ command = "%s %s %s %s" % (SCRIPT, options, CREDS, SERVER)
+ self.check_exit_code(command, 1)
+ command = "%s %s %s %s %s" % (SCRIPT, MODEL, options, CREDS, SERVER)
self.check_exit_code(command, 1)
sys.exit()
# ingest the model
- if model_file:
+ if model_file and not opts.generate_users_only:
try:
model = traffic.TrafficModel()
--- /dev/null
+# This is in flux, so lets fail it for a while
+samba.tests.blackbox.traffic_replay.samba.tests.blackbox.traffic_replay.TrafficLearnerTests.test_summary_generation
git.samba.org / amitay / samba.git / commitdiff