dba2c3f32f9a9768926268dfe63a2f4c11bf82be
[samba.git] / python / samba / emulate / traffic.py
1 # -*- encoding: utf-8 -*-
2 # Samba traffic replay and learning
3 #
4 # Copyright (C) Catalyst IT Ltd. 2017
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
18 #
19 from __future__ import print_function, division
20
21 import time
22 import os
23 import random
24 import json
25 import math
26 import sys
27 import signal
28 import itertools
29
30 from collections import OrderedDict, Counter, defaultdict
31 from samba.emulate import traffic_packets
32 from samba.samdb import SamDB
33 import ldb
34 from ldb import LdbError
35 from samba.dcerpc import ClientConnection
36 from samba.dcerpc import security, drsuapi, lsa
37 from samba.dcerpc import netlogon
38 from samba.dcerpc.netlogon import netr_Authenticator
39 from samba.dcerpc import srvsvc
40 from samba.dcerpc import samr
41 from samba.drs_utils import drs_DsBind
42 import traceback
43 from samba.credentials import Credentials, DONT_USE_KERBEROS, MUST_USE_KERBEROS
44 from samba.auth import system_session
45 from samba.dsdb import (
46     UF_NORMAL_ACCOUNT,
47     UF_SERVER_TRUST_ACCOUNT,
48     UF_TRUSTED_FOR_DELEGATION
49 )
50 from samba.dcerpc.misc import SEC_CHAN_BDC
51 from samba import gensec
52 from samba import sd_utils
53
54 SLEEP_OVERHEAD = 3e-4
55
56 # we don't use None, because it complicates [de]serialisation
57 NON_PACKET = '-'
58
59 CLIENT_CLUES = {
60     ('dns', '0'): 1.0,      # query
61     ('smb', '0x72'): 1.0,   # Negotiate protocol
62     ('ldap', '0'): 1.0,     # bind
63     ('ldap', '3'): 1.0,     # searchRequest
64     ('ldap', '2'): 1.0,     # unbindRequest
65     ('cldap', '3'): 1.0,
66     ('dcerpc', '11'): 1.0,  # bind
67     ('dcerpc', '14'): 1.0,  # Alter_context
68     ('nbns', '0'): 1.0,     # query
69 }
70
71 SERVER_CLUES = {
72     ('dns', '1'): 1.0,      # response
73     ('ldap', '1'): 1.0,     # bind response
74     ('ldap', '4'): 1.0,     # search result
75     ('ldap', '5'): 1.0,     # search done
76     ('cldap', '5'): 1.0,
77     ('dcerpc', '12'): 1.0,  # bind_ack
78     ('dcerpc', '13'): 1.0,  # bind_nak
79     ('dcerpc', '15'): 1.0,  # Alter_context response
80 }
81
82 SKIPPED_PROTOCOLS = {"smb", "smb2", "browser", "smb_netlogon"}
83
84 WAIT_SCALE = 10.0
85 WAIT_THRESHOLD = (1.0 / WAIT_SCALE)
86 NO_WAIT_LOG_TIME_RANGE = (-10, -3)
87
88 # DEBUG_LEVEL can be changed by scripts with -d
89 DEBUG_LEVEL = 0
90
91
92 def debug(level, msg, *args):
93     """Print a formatted debug message to standard error.
94
95
96     :param level: The debug level, message will be printed if it is <= the
97                   currently set debug level. The debug level can be set with
98                   the -d option.
99     :param msg:   The message to be logged, can contain C-Style format
100                   specifiers
101     :param args:  The parameters required by the format specifiers
102     """
103     if level <= DEBUG_LEVEL:
104         if not args:
105             print(msg, file=sys.stderr)
106         else:
107             print(msg % tuple(args), file=sys.stderr)
108
109
110 def debug_lineno(*args):
111     """ Print an unformatted log message to stderr, contaning the line number
112     """
113     tb = traceback.extract_stack(limit=2)
114     print((" %s:" "\033[01;33m"
115            "%s " "\033[00m" % (tb[0][2], tb[0][1])), end=' ',
116           file=sys.stderr)
117     for a in args:
118         print(a, file=sys.stderr)
119     print(file=sys.stderr)
120     sys.stderr.flush()
121
122
123 def random_colour_print():
124     """Return a function that prints a randomly coloured line to stderr"""
125     n = 18 + random.randrange(214)
126     prefix = "\033[38;5;%dm" % n
127
128     def p(*args):
129         for a in args:
130             print("%s%s\033[00m" % (prefix, a), file=sys.stderr)
131
132     return p
133
134
135 class FakePacketError(Exception):
136     pass
137
138
139 class Packet(object):
140     """Details of a network packet"""
141     def __init__(self, timestamp, ip_protocol, stream_number, src, dest,
142                  protocol, opcode, desc, extra):
143
144         self.timestamp = timestamp
145         self.ip_protocol = ip_protocol
146         self.stream_number = stream_number
147         self.src = src
148         self.dest = dest
149         self.protocol = protocol
150         self.opcode = opcode
151         self.desc = desc
152         self.extra = extra
153         if self.src < self.dest:
154             self.endpoints = (self.src, self.dest)
155         else:
156             self.endpoints = (self.dest, self.src)
157
158     @classmethod
159     def from_line(self, line):
160         fields = line.rstrip('\n').split('\t')
161         (timestamp,
162          ip_protocol,
163          stream_number,
164          src,
165          dest,
166          protocol,
167          opcode,
168          desc) = fields[:8]
169         extra = fields[8:]
170
171         timestamp = float(timestamp)
172         src = int(src)
173         dest = int(dest)
174
175         return Packet(timestamp, ip_protocol, stream_number, src, dest,
176                       protocol, opcode, desc, extra)
177
178     def as_summary(self, time_offset=0.0):
179         """Format the packet as a traffic_summary line.
180         """
181         extra = '\t'.join(self.extra)
182         t = self.timestamp + time_offset
183         return (t, '%f\t%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s' %
184                 (t,
185                  self.ip_protocol,
186                  self.stream_number or '',
187                  self.src,
188                  self.dest,
189                  self.protocol,
190                  self.opcode,
191                  self.desc,
192                  extra))
193
194     def __str__(self):
195         return ("%.3f: %d -> %d; ip %s; strm %s; prot %s; op %s; desc %s %s" %
196                 (self.timestamp, self.src, self.dest, self.ip_protocol or '-',
197                  self.stream_number, self.protocol, self.opcode, self.desc,
198                  ('«' + ' '.join(self.extra) + '»' if self.extra else '')))
199
200     def __repr__(self):
201         return "<Packet @%s>" % self
202
203     def copy(self):
204         return self.__class__(self.timestamp,
205                               self.ip_protocol,
206                               self.stream_number,
207                               self.src,
208                               self.dest,
209                               self.protocol,
210                               self.opcode,
211                               self.desc,
212                               self.extra)
213
214     def as_packet_type(self):
215         t = '%s:%s' % (self.protocol, self.opcode)
216         return t
217
218     def client_score(self):
219         """A positive number means we think it is a client; a negative number
220         means we think it is a server. Zero means no idea. range: -1 to 1.
221         """
222         key = (self.protocol, self.opcode)
223         if key in CLIENT_CLUES:
224             return CLIENT_CLUES[key]
225         if key in SERVER_CLUES:
226             return -SERVER_CLUES[key]
227         return 0.0
228
229     def play(self, conversation, context):
230         """Send the packet over the network, if required.
231
232         Some packets are ignored, i.e. for  protocols not handled,
233         server response messages, or messages that are generated by the
234         protocol layer associated with other packets.
235         """
236         fn_name = 'packet_%s_%s' % (self.protocol, self.opcode)
237         try:
238             fn = getattr(traffic_packets, fn_name)
239
240         except AttributeError as e:
241             print("Conversation(%s) Missing handler %s" % \
242                   (conversation.conversation_id, fn_name),
243                   file=sys.stderr)
244             return
245
246         # Don't display a message for kerberos packets, they're not directly
247         # generated they're used to indicate kerberos should be used
248         if self.protocol != "kerberos":
249             debug(2, "Conversation(%s) Calling handler %s" %
250                      (conversation.conversation_id, fn_name))
251
252         start = time.time()
253         try:
254             if fn(self, conversation, context):
255                 # Only collect timing data for functions that generate
256                 # network traffic, or fail
257                 end = time.time()
258                 duration = end - start
259                 print("%f\t%s\t%s\t%s\t%f\tTrue\t" %
260                       (end, conversation.conversation_id, self.protocol,
261                        self.opcode, duration))
262         except Exception as e:
263             end = time.time()
264             duration = end - start
265             print("%f\t%s\t%s\t%s\t%f\tFalse\t%s" %
266                   (end, conversation.conversation_id, self.protocol,
267                    self.opcode, duration, e))
268
269     def __cmp__(self, other):
270         return self.timestamp - other.timestamp
271
272     def is_really_a_packet(self, missing_packet_stats=None):
273         """Is the packet one that can be ignored?
274
275         If so removing it will have no effect on the replay
276         """
277         if self.protocol in SKIPPED_PROTOCOLS:
278             # Ignore any packets for the protocols we're not interested in.
279             return False
280         if self.protocol == "ldap" and self.opcode == '':
281             # skip ldap continuation packets
282             return False
283
284         fn_name = 'packet_%s_%s' % (self.protocol, self.opcode)
285         fn = getattr(traffic_packets, fn_name, None)
286         if not fn:
287             print("missing packet %s" % fn_name, file=sys.stderr)
288             return False
289         if fn is traffic_packets.null_packet:
290             return False
291         return True
292
293
294 class ReplayContext(object):
295     """State/Context for an individual conversation between an simulated client
296        and a server.
297     """
298
299     def __init__(self,
300                  server=None,
301                  lp=None,
302                  creds=None,
303                  badpassword_frequency=None,
304                  prefer_kerberos=None,
305                  tempdir=None,
306                  statsdir=None,
307                  ou=None,
308                  base_dn=None,
309                  domain=None,
310                  domain_sid=None):
311
312         self.server                   = server
313         self.ldap_connections         = []
314         self.dcerpc_connections       = []
315         self.lsarpc_connections       = []
316         self.lsarpc_connections_named = []
317         self.drsuapi_connections      = []
318         self.srvsvc_connections       = []
319         self.samr_contexts            = []
320         self.netlogon_connection      = None
321         self.creds                    = creds
322         self.lp                       = lp
323         self.prefer_kerberos          = prefer_kerberos
324         self.ou                       = ou
325         self.base_dn                  = base_dn
326         self.domain                   = domain
327         self.statsdir                 = statsdir
328         self.global_tempdir           = tempdir
329         self.domain_sid               = domain_sid
330         self.realm                    = lp.get('realm')
331
332         # Bad password attempt controls
333         self.badpassword_frequency    = badpassword_frequency
334         self.last_lsarpc_bad          = False
335         self.last_lsarpc_named_bad    = False
336         self.last_simple_bind_bad     = False
337         self.last_bind_bad            = False
338         self.last_srvsvc_bad          = False
339         self.last_drsuapi_bad         = False
340         self.last_netlogon_bad        = False
341         self.last_samlogon_bad        = False
342         self.generate_ldap_search_tables()
343         self.next_conversation_id = next(itertools.count())
344
345     def generate_ldap_search_tables(self):
346         session = system_session()
347
348         db = SamDB(url="ldap://%s" % self.server,
349                    session_info=session,
350                    credentials=self.creds,
351                    lp=self.lp)
352
353         res = db.search(db.domain_dn(),
354                         scope=ldb.SCOPE_SUBTREE,
355                         controls=["paged_results:1:1000"],
356                         attrs=['dn'])
357
358         # find a list of dns for each pattern
359         # e.g. CN,CN,CN,DC,DC
360         dn_map = {}
361         attribute_clue_map = {
362             'invocationId': []
363         }
364
365         for r in res:
366             dn = str(r.dn)
367             pattern = ','.join(x.lstrip()[:2] for x in dn.split(',')).upper()
368             dns = dn_map.setdefault(pattern, [])
369             dns.append(dn)
370             if dn.startswith('CN=NTDS Settings,'):
371                 attribute_clue_map['invocationId'].append(dn)
372
373         # extend the map in case we are working with a different
374         # number of DC components.
375         # for k, v in self.dn_map.items():
376         #     print >>sys.stderr, k, len(v)
377
378         for k in list(dn_map.keys()):
379             if k[-3:] != ',DC':
380                 continue
381             p = k[:-3]
382             while p[-3:] == ',DC':
383                 p = p[:-3]
384             for i in range(5):
385                 p += ',DC'
386                 if p != k and p in dn_map:
387                     print('dn_map collison %s %s' % (k, p),
388                           file=sys.stderr)
389                     continue
390                 dn_map[p] = dn_map[k]
391
392         self.dn_map = dn_map
393         self.attribute_clue_map = attribute_clue_map
394
395     def generate_process_local_config(self, account, conversation):
396         if account is None:
397             return
398         self.netbios_name             = account.netbios_name
399         self.machinepass              = account.machinepass
400         self.username                 = account.username
401         self.userpass                 = account.userpass
402
403         self.tempdir = mk_masked_dir(self.global_tempdir,
404                                      'conversation-%d' %
405                                      conversation.conversation_id)
406
407         self.lp.set("private dir",     self.tempdir)
408         self.lp.set("lock dir",        self.tempdir)
409         self.lp.set("state directory", self.tempdir)
410         self.lp.set("tls verify peer", "no_check")
411
412         # If the domain was not specified, check for the environment
413         # variable.
414         if self.domain is None:
415             self.domain = os.environ["DOMAIN"]
416
417         self.remoteAddress = "/root/ncalrpc_as_system"
418         self.samlogon_dn   = ("cn=%s,%s" %
419                               (self.netbios_name, self.ou))
420         self.user_dn       = ("cn=%s,%s" %
421                               (self.username, self.ou))
422
423         self.generate_machine_creds()
424         self.generate_user_creds()
425
426     def with_random_bad_credentials(self, f, good, bad, failed_last_time):
427         """Execute the supplied logon function, randomly choosing the
428            bad credentials.
429
430            Based on the frequency in badpassword_frequency randomly perform the
431            function with the supplied bad credentials.
432            If run with bad credentials, the function is re-run with the good
433            credentials.
434            failed_last_time is used to prevent consecutive bad credential
435            attempts. So the over all bad credential frequency will be lower
436            than that requested, but not significantly.
437         """
438         if not failed_last_time:
439             if (self.badpassword_frequency > 0 and
440                random.random() < self.badpassword_frequency):
441                 try:
442                     f(bad)
443                 except:
444                     # Ignore any exceptions as the operation may fail
445                     # as it's being performed with bad credentials
446                     pass
447                 failed_last_time = True
448             else:
449                 failed_last_time = False
450
451         result = f(good)
452         return (result, failed_last_time)
453
454     def generate_user_creds(self):
455         """Generate the conversation specific user Credentials.
456
457         Each Conversation has an associated user account used to simulate
458         any non Administrative user traffic.
459
460         Generates user credentials with good and bad passwords and ldap
461         simple bind credentials with good and bad passwords.
462         """
463         self.user_creds = Credentials()
464         self.user_creds.guess(self.lp)
465         self.user_creds.set_workstation(self.netbios_name)
466         self.user_creds.set_password(self.userpass)
467         self.user_creds.set_username(self.username)
468         self.user_creds.set_domain(self.domain)
469         if self.prefer_kerberos:
470             self.user_creds.set_kerberos_state(MUST_USE_KERBEROS)
471         else:
472             self.user_creds.set_kerberos_state(DONT_USE_KERBEROS)
473
474         self.user_creds_bad = Credentials()
475         self.user_creds_bad.guess(self.lp)
476         self.user_creds_bad.set_workstation(self.netbios_name)
477         self.user_creds_bad.set_password(self.userpass[:-4])
478         self.user_creds_bad.set_username(self.username)
479         if self.prefer_kerberos:
480             self.user_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
481         else:
482             self.user_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
483
484         # Credentials for ldap simple bind.
485         self.simple_bind_creds = Credentials()
486         self.simple_bind_creds.guess(self.lp)
487         self.simple_bind_creds.set_workstation(self.netbios_name)
488         self.simple_bind_creds.set_password(self.userpass)
489         self.simple_bind_creds.set_username(self.username)
490         self.simple_bind_creds.set_gensec_features(
491             self.simple_bind_creds.get_gensec_features() | gensec.FEATURE_SEAL)
492         if self.prefer_kerberos:
493             self.simple_bind_creds.set_kerberos_state(MUST_USE_KERBEROS)
494         else:
495             self.simple_bind_creds.set_kerberos_state(DONT_USE_KERBEROS)
496         self.simple_bind_creds.set_bind_dn(self.user_dn)
497
498         self.simple_bind_creds_bad = Credentials()
499         self.simple_bind_creds_bad.guess(self.lp)
500         self.simple_bind_creds_bad.set_workstation(self.netbios_name)
501         self.simple_bind_creds_bad.set_password(self.userpass[:-4])
502         self.simple_bind_creds_bad.set_username(self.username)
503         self.simple_bind_creds_bad.set_gensec_features(
504             self.simple_bind_creds_bad.get_gensec_features() |
505             gensec.FEATURE_SEAL)
506         if self.prefer_kerberos:
507             self.simple_bind_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
508         else:
509             self.simple_bind_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
510         self.simple_bind_creds_bad.set_bind_dn(self.user_dn)
511
512     def generate_machine_creds(self):
513         """Generate the conversation specific machine Credentials.
514
515         Each Conversation has an associated machine account.
516
517         Generates machine credentials with good and bad passwords.
518         """
519
520         self.machine_creds = Credentials()
521         self.machine_creds.guess(self.lp)
522         self.machine_creds.set_workstation(self.netbios_name)
523         self.machine_creds.set_secure_channel_type(SEC_CHAN_BDC)
524         self.machine_creds.set_password(self.machinepass)
525         self.machine_creds.set_username(self.netbios_name + "$")
526         self.machine_creds.set_domain(self.domain)
527         if self.prefer_kerberos:
528             self.machine_creds.set_kerberos_state(MUST_USE_KERBEROS)
529         else:
530             self.machine_creds.set_kerberos_state(DONT_USE_KERBEROS)
531
532         self.machine_creds_bad = Credentials()
533         self.machine_creds_bad.guess(self.lp)
534         self.machine_creds_bad.set_workstation(self.netbios_name)
535         self.machine_creds_bad.set_secure_channel_type(SEC_CHAN_BDC)
536         self.machine_creds_bad.set_password(self.machinepass[:-4])
537         self.machine_creds_bad.set_username(self.netbios_name + "$")
538         if self.prefer_kerberos:
539             self.machine_creds_bad.set_kerberos_state(MUST_USE_KERBEROS)
540         else:
541             self.machine_creds_bad.set_kerberos_state(DONT_USE_KERBEROS)
542
543     def get_matching_dn(self, pattern, attributes=None):
544         # If the pattern is an empty string, we assume ROOTDSE,
545         # Otherwise we try adding or removing DC suffixes, then
546         # shorter leading patterns until we hit one.
547         # e.g if there is no CN,CN,CN,CN,DC,DC
548         # we first try       CN,CN,CN,CN,DC
549         # and                CN,CN,CN,CN,DC,DC,DC
550         # then change to        CN,CN,CN,DC,DC
551         # and as last resort we use the base_dn
552         attr_clue = self.attribute_clue_map.get(attributes)
553         if attr_clue:
554             return random.choice(attr_clue)
555
556         pattern = pattern.upper()
557         while pattern:
558             if pattern in self.dn_map:
559                 return random.choice(self.dn_map[pattern])
560             # chop one off the front and try it all again.
561             pattern = pattern[3:]
562
563         return self.base_dn
564
565     def get_dcerpc_connection(self, new=False):
566         guid = '12345678-1234-abcd-ef00-01234567cffb'  # RPC_NETLOGON UUID
567         if self.dcerpc_connections and not new:
568             return self.dcerpc_connections[-1]
569         c = ClientConnection("ncacn_ip_tcp:%s" % self.server,
570                              (guid, 1), self.lp)
571         self.dcerpc_connections.append(c)
572         return c
573
574     def get_srvsvc_connection(self, new=False):
575         if self.srvsvc_connections and not new:
576             return self.srvsvc_connections[-1]
577
578         def connect(creds):
579             return srvsvc.srvsvc("ncacn_np:%s" % (self.server),
580                                  self.lp,
581                                  creds)
582
583         (c, self.last_srvsvc_bad) = \
584             self.with_random_bad_credentials(connect,
585                                              self.user_creds,
586                                              self.user_creds_bad,
587                                              self.last_srvsvc_bad)
588
589         self.srvsvc_connections.append(c)
590         return c
591
592     def get_lsarpc_connection(self, new=False):
593         if self.lsarpc_connections and not new:
594             return self.lsarpc_connections[-1]
595
596         def connect(creds):
597             binding_options = 'schannel,seal,sign'
598             return lsa.lsarpc("ncacn_ip_tcp:%s[%s]" %
599                               (self.server, binding_options),
600                               self.lp,
601                               creds)
602
603         (c, self.last_lsarpc_bad) = \
604             self.with_random_bad_credentials(connect,
605                                              self.machine_creds,
606                                              self.machine_creds_bad,
607                                              self.last_lsarpc_bad)
608
609         self.lsarpc_connections.append(c)
610         return c
611
612     def get_lsarpc_named_pipe_connection(self, new=False):
613         if self.lsarpc_connections_named and not new:
614             return self.lsarpc_connections_named[-1]
615
616         def connect(creds):
617             return lsa.lsarpc("ncacn_np:%s" % (self.server),
618                               self.lp,
619                               creds)
620
621         (c, self.last_lsarpc_named_bad) = \
622             self.with_random_bad_credentials(connect,
623                                              self.machine_creds,
624                                              self.machine_creds_bad,
625                                              self.last_lsarpc_named_bad)
626
627         self.lsarpc_connections_named.append(c)
628         return c
629
630     def get_drsuapi_connection_pair(self, new=False, unbind=False):
631         """get a (drs, drs_handle) tuple"""
632         if self.drsuapi_connections and not new:
633             c = self.drsuapi_connections[-1]
634             return c
635
636         def connect(creds):
637             binding_options = 'seal'
638             binding_string = "ncacn_ip_tcp:%s[%s]" %\
639                              (self.server, binding_options)
640             return drsuapi.drsuapi(binding_string, self.lp, creds)
641
642         (drs, self.last_drsuapi_bad) = \
643             self.with_random_bad_credentials(connect,
644                                              self.user_creds,
645                                              self.user_creds_bad,
646                                              self.last_drsuapi_bad)
647
648         (drs_handle, supported_extensions) = drs_DsBind(drs)
649         c = (drs, drs_handle)
650         self.drsuapi_connections.append(c)
651         return c
652
653     def get_ldap_connection(self, new=False, simple=False):
654         if self.ldap_connections and not new:
655             return self.ldap_connections[-1]
656
657         def simple_bind(creds):
658             """
659             To run simple bind against Windows, we need to run
660             following commands in PowerShell:
661
662                 Install-windowsfeature ADCS-Cert-Authority
663                 Install-AdcsCertificationAuthority -CAType EnterpriseRootCA
664                 Restart-Computer
665
666             """
667             return SamDB('ldaps://%s' % self.server,
668                          credentials=creds,
669                          lp=self.lp)
670
671         def sasl_bind(creds):
672             return SamDB('ldap://%s' % self.server,
673                          credentials=creds,
674                          lp=self.lp)
675         if simple:
676             (samdb, self.last_simple_bind_bad) = \
677                 self.with_random_bad_credentials(simple_bind,
678                                                  self.simple_bind_creds,
679                                                  self.simple_bind_creds_bad,
680                                                  self.last_simple_bind_bad)
681         else:
682             (samdb, self.last_bind_bad) = \
683                 self.with_random_bad_credentials(sasl_bind,
684                                                  self.user_creds,
685                                                  self.user_creds_bad,
686                                                  self.last_bind_bad)
687
688         self.ldap_connections.append(samdb)
689         return samdb
690
691     def get_samr_context(self, new=False):
692         if not self.samr_contexts or new:
693             self.samr_contexts.append(
694                 SamrContext(self.server, lp=self.lp, creds=self.creds))
695         return self.samr_contexts[-1]
696
697     def get_netlogon_connection(self):
698
699         if self.netlogon_connection:
700             return self.netlogon_connection
701
702         def connect(creds):
703             return netlogon.netlogon("ncacn_ip_tcp:%s[schannel,seal]" %
704                                      (self.server),
705                                      self.lp,
706                                      creds)
707         (c, self.last_netlogon_bad) = \
708             self.with_random_bad_credentials(connect,
709                                              self.machine_creds,
710                                              self.machine_creds_bad,
711                                              self.last_netlogon_bad)
712         self.netlogon_connection = c
713         return c
714
715     def guess_a_dns_lookup(self):
716         return (self.realm, 'A')
717
718     def get_authenticator(self):
719         auth = self.machine_creds.new_client_authenticator()
720         current  = netr_Authenticator()
721         current.cred.data = [ord(x) for x in auth["credential"]]
722         current.timestamp = auth["timestamp"]
723
724         subsequent = netr_Authenticator()
725         return (current, subsequent)
726
727
728 class SamrContext(object):
729     """State/Context associated with a samr connection.
730     """
731     def __init__(self, server, lp=None, creds=None):
732         self.connection    = None
733         self.handle        = None
734         self.domain_handle = None
735         self.domain_sid    = None
736         self.group_handle  = None
737         self.user_handle   = None
738         self.rids          = None
739         self.server        = server
740         self.lp            = lp
741         self.creds         = creds
742
743     def get_connection(self):
744         if not self.connection:
745             self.connection = samr.samr(
746                 "ncacn_ip_tcp:%s[seal]" % (self.server),
747                 lp_ctx=self.lp,
748                 credentials=self.creds)
749
750         return self.connection
751
752     def get_handle(self):
753         if not self.handle:
754             c = self.get_connection()
755             self.handle = c.Connect2(None, security.SEC_FLAG_MAXIMUM_ALLOWED)
756         return self.handle
757
758
759 class Conversation(object):
760     """Details of a converation between a simulated client and a server."""
761     conversation_id = None
762
763     def __init__(self, start_time=None, endpoints=None):
764         self.start_time = start_time
765         self.endpoints = endpoints
766         self.packets = []
767         self.msg = random_colour_print()
768         self.client_balance = 0.0
769
770     def __cmp__(self, other):
771         if self.start_time is None:
772             if other.start_time is None:
773                 return 0
774             return -1
775         if other.start_time is None:
776             return 1
777         return self.start_time - other.start_time
778
779     def add_packet(self, packet):
780         """Add a packet object to this conversation, making a local copy with
781         a conversation-relative timestamp."""
782         p = packet.copy()
783
784         if self.start_time is None:
785             self.start_time = p.timestamp
786
787         if self.endpoints is None:
788             self.endpoints = p.endpoints
789
790         if p.endpoints != self.endpoints:
791             raise FakePacketError("Conversation endpoints %s don't match"
792                                   "packet endpoints %s" %
793                                   (self.endpoints, p.endpoints))
794
795         p.timestamp -= self.start_time
796
797         if p.src == p.endpoints[0]:
798             self.client_balance -= p.client_score()
799         else:
800             self.client_balance += p.client_score()
801
802         if p.is_really_a_packet():
803             self.packets.append(p)
804
805     def add_short_packet(self, timestamp, protocol, opcode, extra,
806                          client=True):
807         """Create a packet from a timestamp, and 'protocol:opcode' pair, and a
808         (possibly empty) list of extra data. If client is True, assume
809         this packet is from the client to the server.
810         """
811         src, dest = self.guess_client_server()
812         if not client:
813             src, dest = dest, src
814         key = (protocol, opcode)
815         desc = OP_DESCRIPTIONS[key] if key in OP_DESCRIPTIONS else ''
816         if protocol in IP_PROTOCOLS:
817             ip_protocol = IP_PROTOCOLS[protocol]
818         else:
819             ip_protocol = '06'
820         packet = Packet(timestamp - self.start_time, ip_protocol,
821                         '', src, dest,
822                         protocol, opcode, desc, extra)
823         # XXX we're assuming the timestamp is already adjusted for
824         # this conversation?
825         # XXX should we adjust client balance for guessed packets?
826         if packet.src == packet.endpoints[0]:
827             self.client_balance -= packet.client_score()
828         else:
829             self.client_balance += packet.client_score()
830         if packet.is_really_a_packet():
831             self.packets.append(packet)
832
833     def __str__(self):
834         return ("<Conversation %s %s starting %.3f %d packets>" %
835                 (self.conversation_id, self.endpoints, self.start_time,
836                  len(self.packets)))
837
838     __repr__ = __str__
839
840     def __iter__(self):
841         return iter(self.packets)
842
843     def __len__(self):
844         return len(self.packets)
845
846     def get_duration(self):
847         if len(self.packets) < 2:
848             return 0
849         return self.packets[-1].timestamp - self.packets[0].timestamp
850
851     def replay_as_summary_lines(self):
852         lines = []
853         for p in self.packets:
854             lines.append(p.as_summary(self.start_time))
855         return lines
856
857     def replay_in_fork_with_delay(self, start, context=None, account=None):
858         """Fork a new process and replay the conversation.
859         """
860         def signal_handler(signal, frame):
861             """Signal handler closes standard out and error.
862
863             Triggered by a sigterm, ensures that the log messages are flushed
864             to disk and not lost.
865             """
866             sys.stderr.close()
867             sys.stdout.close()
868             os._exit(0)
869
870         t = self.start_time
871         now = time.time() - start
872         gap = t - now
873         # we are replaying strictly in order, so it is safe to sleep
874         # in the main process if the gap is big enough. This reduces
875         # the number of concurrent threads, which allows us to make
876         # larger loads.
877         if gap > 0.15 and False:
878             print("sleeping for %f in main process" % (gap - 0.1),
879                   file=sys.stderr)
880             time.sleep(gap - 0.1)
881             now = time.time() - start
882             gap = t - now
883             print("gap is now %f" % gap, file=sys.stderr)
884
885         self.conversation_id = context.next_conversation_id()
886         pid = os.fork()
887         if pid != 0:
888             return pid
889         pid = os.getpid()
890         signal.signal(signal.SIGTERM, signal_handler)
891         # we must never return, or we'll end up running parts of the
892         # parent's clean-up code. So we work in a try...finally, and
893         # try to print any exceptions.
894
895         try:
896             context.generate_process_local_config(account, self)
897             sys.stdin.close()
898             os.close(0)
899             filename = os.path.join(context.statsdir, 'stats-conversation-%d' %
900                                     self.conversation_id)
901             sys.stdout.close()
902             sys.stdout = open(filename, 'w')
903
904             sleep_time = gap - SLEEP_OVERHEAD
905             if sleep_time > 0:
906                 time.sleep(sleep_time)
907
908             miss = t - (time.time() - start)
909             self.msg("starting %s [miss %.3f pid %d]" % (self, miss, pid))
910             self.replay(context)
911         except Exception:
912             print(("EXCEPTION in child PID %d, conversation %s" % (pid, self)),
913                   file=sys.stderr)
914             traceback.print_exc(sys.stderr)
915         finally:
916             sys.stderr.close()
917             sys.stdout.close()
918             os._exit(0)
919
920     def replay(self, context=None):
921         start = time.time()
922
923         for p in self.packets:
924             now = time.time() - start
925             gap = p.timestamp - now
926             sleep_time = gap - SLEEP_OVERHEAD
927             if sleep_time > 0:
928                 time.sleep(sleep_time)
929
930             miss = p.timestamp - (time.time() - start)
931             if context is None:
932                 self.msg("packet %s [miss %.3f pid %d]" % (p, miss,
933                                                            os.getpid()))
934                 continue
935             p.play(self, context)
936
937     def guess_client_server(self, server_clue=None):
938         """Have a go at deciding who is the server and who is the client.
939         returns (client, server)
940         """
941         a, b = self.endpoints
942
943         if self.client_balance < 0:
944             return (a, b)
945
946         # in the absense of a clue, we will fall through to assuming
947         # the lowest number is the server (which is usually true).
948
949         if self.client_balance == 0 and server_clue == b:
950             return (a, b)
951
952         return (b, a)
953
954     def forget_packets_outside_window(self, s, e):
955         """Prune any packets outside the timne window we're interested in
956
957         :param s: start of the window
958         :param e: end of the window
959         """
960         self.packets = [p for p in self.packets if s <= p.timestamp <= e]
961         self.start_time = self.packets[0].timestamp if self.packets else None
962
963     def renormalise_times(self, start_time):
964         """Adjust the packet start times relative to the new start time."""
965         for p in self.packets:
966             p.timestamp -= start_time
967
968         if self.start_time is not None:
969             self.start_time -= start_time
970
971
972 class DnsHammer(Conversation):
973     """A lightweight conversation that generates a lot of dns:0 packets on
974     the fly"""
975
976     def __init__(self, dns_rate, duration):
977         n = int(dns_rate * duration)
978         self.times = [random.uniform(0, duration) for i in range(n)]
979         self.times.sort()
980         self.rate = dns_rate
981         self.duration = duration
982         self.start_time = 0
983         self.msg = random_colour_print()
984
985     def __str__(self):
986         return ("<DnsHammer %d packets over %.1fs (rate %.2f)>" %
987                 (len(self.times), self.duration, self.rate))
988
989     def replay_in_fork_with_delay(self, start, context=None, account=None):
990         return Conversation.replay_in_fork_with_delay(self,
991                                                       start,
992                                                       context,
993                                                       account)
994
995     def replay(self, context=None):
996         start = time.time()
997         fn = traffic_packets.packet_dns_0
998         for t in self.times:
999             now = time.time() - start
1000             gap = t - now
1001             sleep_time = gap - SLEEP_OVERHEAD
1002             if sleep_time > 0:
1003                 time.sleep(sleep_time)
1004
1005             if context is None:
1006                 miss = t - (time.time() - start)
1007                 self.msg("packet %s [miss %.3f pid %d]" % (t, miss,
1008                                                            os.getpid()))
1009                 continue
1010
1011             packet_start = time.time()
1012             try:
1013                 fn(self, self, context)
1014                 end = time.time()
1015                 duration = end - packet_start
1016                 print("%f\tDNS\tdns\t0\t%f\tTrue\t" % (end, duration))
1017             except Exception as e:
1018                 end = time.time()
1019                 duration = end - packet_start
1020                 print("%f\tDNS\tdns\t0\t%f\tFalse\t%s" % (end, duration, e))
1021
1022
1023 def ingest_summaries(files, dns_mode='count'):
1024     """Load a summary traffic summary file and generated Converations from it.
1025     """
1026
1027     dns_counts = defaultdict(int)
1028     packets = []
1029     for f in files:
1030         if isinstance(f, str):
1031             f = open(f)
1032         print("Ingesting %s" % (f.name,), file=sys.stderr)
1033         for line in f:
1034             p = Packet.from_line(line)
1035             if p.protocol == 'dns' and dns_mode != 'include':
1036                 dns_counts[p.opcode] += 1
1037             else:
1038                 packets.append(p)
1039
1040         f.close()
1041
1042     if not packets:
1043         return [], 0
1044
1045     start_time = min(p.timestamp for p in packets)
1046     last_packet = max(p.timestamp for p in packets)
1047
1048     print("gathering packets into conversations", file=sys.stderr)
1049     conversations = OrderedDict()
1050     for p in packets:
1051         p.timestamp -= start_time
1052         c = conversations.get(p.endpoints)
1053         if c is None:
1054             c = Conversation()
1055             conversations[p.endpoints] = c
1056         c.add_packet(p)
1057
1058     # We only care about conversations with actual traffic, so we
1059     # filter out conversations with nothing to say. We do that here,
1060     # rather than earlier, because those empty packets contain useful
1061     # hints as to which end of the conversation was the client.
1062     conversation_list = []
1063     for c in conversations.values():
1064         if len(c) != 0:
1065             conversation_list.append(c)
1066
1067     # This is obviously not correct, as many conversations will appear
1068     # to start roughly simultaneously at the beginning of the snapshot.
1069     # To which we say: oh well, so be it.
1070     duration = float(last_packet - start_time)
1071     mean_interval = len(conversations) / duration
1072
1073     return conversation_list, mean_interval, duration, dns_counts
1074
1075
1076 def guess_server_address(conversations):
1077     # we guess the most common address.
1078     addresses = Counter()
1079     for c in conversations:
1080         addresses.update(c.endpoints)
1081     if addresses:
1082         return addresses.most_common(1)[0]
1083
1084
1085 def stringify_keys(x):
1086     y = {}
1087     for k, v in x.items():
1088         k2 = '\t'.join(k)
1089         y[k2] = v
1090     return y
1091
1092
1093 def unstringify_keys(x):
1094     y = {}
1095     for k, v in x.items():
1096         t = tuple(str(k).split('\t'))
1097         y[t] = v
1098     return y
1099
1100
1101 class TrafficModel(object):
1102     def __init__(self, n=3):
1103         self.ngrams = {}
1104         self.query_details = {}
1105         self.n = n
1106         self.dns_opcounts = defaultdict(int)
1107         self.cumulative_duration = 0.0
1108         self.conversation_rate = [0, 1]
1109
1110     def learn(self, conversations, dns_opcounts={}):
1111         prev = 0.0
1112         cum_duration = 0.0
1113         key = (NON_PACKET,) * (self.n - 1)
1114
1115         server = guess_server_address(conversations)
1116
1117         for k, v in dns_opcounts.items():
1118             self.dns_opcounts[k] += v
1119
1120         if len(conversations) > 1:
1121             elapsed =\
1122                 conversations[-1].start_time - conversations[0].start_time
1123             self.conversation_rate[0] = len(conversations)
1124             self.conversation_rate[1] = elapsed
1125
1126         for c in conversations:
1127             client, server = c.guess_client_server(server)
1128             cum_duration += c.get_duration()
1129             key = (NON_PACKET,) * (self.n - 1)
1130             for p in c:
1131                 if p.src != client:
1132                     continue
1133
1134                 elapsed = p.timestamp - prev
1135                 prev = p.timestamp
1136                 if elapsed > WAIT_THRESHOLD:
1137                     # add the wait as an extra state
1138                     wait = 'wait:%d' % (math.log(max(1.0,
1139                                                      elapsed * WAIT_SCALE)))
1140                     self.ngrams.setdefault(key, []).append(wait)
1141                     key = key[1:] + (wait,)
1142
1143                 short_p = p.as_packet_type()
1144                 self.query_details.setdefault(short_p,
1145                                               []).append(tuple(p.extra))
1146                 self.ngrams.setdefault(key, []).append(short_p)
1147                 key = key[1:] + (short_p,)
1148
1149         self.cumulative_duration += cum_duration
1150         # add in the end
1151         self.ngrams.setdefault(key, []).append(NON_PACKET)
1152
1153     def save(self, f):
1154         ngrams = {}
1155         for k, v in self.ngrams.items():
1156             k = '\t'.join(k)
1157             ngrams[k] = dict(Counter(v))
1158
1159         query_details = {}
1160         for k, v in self.query_details.items():
1161             query_details[k] = dict(Counter('\t'.join(x) if x else '-'
1162                                             for x in v))
1163
1164         d = {
1165             'ngrams': ngrams,
1166             'query_details': query_details,
1167             'cumulative_duration': self.cumulative_duration,
1168             'conversation_rate': self.conversation_rate,
1169         }
1170         d['dns'] = self.dns_opcounts
1171
1172         if isinstance(f, str):
1173             f = open(f, 'w')
1174
1175         json.dump(d, f, indent=2)
1176
1177     def load(self, f):
1178         if isinstance(f, str):
1179             f = open(f)
1180
1181         d = json.load(f)
1182
1183         for k, v in d['ngrams'].items():
1184             k = tuple(str(k).split('\t'))
1185             values = self.ngrams.setdefault(k, [])
1186             for p, count in v.items():
1187                 values.extend([str(p)] * count)
1188
1189         for k, v in d['query_details'].items():
1190             values = self.query_details.setdefault(str(k), [])
1191             for p, count in v.items():
1192                 if p == '-':
1193                     values.extend([()] * count)
1194                 else:
1195                     values.extend([tuple(str(p).split('\t'))] * count)
1196
1197         if 'dns' in d:
1198             for k, v in d['dns'].items():
1199                 self.dns_opcounts[k] += v
1200
1201         self.cumulative_duration = d['cumulative_duration']
1202         self.conversation_rate = d['conversation_rate']
1203
1204     def construct_conversation(self, timestamp=0.0, client=2, server=1,
1205                                hard_stop=None, packet_rate=1):
1206         """Construct a individual converation from the model."""
1207
1208         c = Conversation(timestamp, (server, client))
1209
1210         key = (NON_PACKET,) * (self.n - 1)
1211
1212         while key in self.ngrams:
1213             p = random.choice(self.ngrams.get(key, NON_PACKET))
1214             if p == NON_PACKET:
1215                 break
1216             if p in self.query_details:
1217                 extra = random.choice(self.query_details[p])
1218             else:
1219                 extra = []
1220
1221             protocol, opcode = p.split(':', 1)
1222             if protocol == 'wait':
1223                 log_wait_time = int(opcode) + random.random()
1224                 wait = math.exp(log_wait_time) / (WAIT_SCALE * packet_rate)
1225                 timestamp += wait
1226             else:
1227                 log_wait = random.uniform(*NO_WAIT_LOG_TIME_RANGE)
1228                 wait = math.exp(log_wait) / packet_rate
1229                 timestamp += wait
1230                 if hard_stop is not None and timestamp > hard_stop:
1231                     break
1232                 c.add_short_packet(timestamp, protocol, opcode, extra)
1233
1234             key = key[1:] + (p,)
1235
1236         return c
1237
1238     def generate_conversations(self, rate, duration, packet_rate=1):
1239         """Generate a list of conversations from the model."""
1240
1241         # We run the simulation for at least ten times as long as our
1242         # desired duration, and take a section near the start.
1243         rate_n, rate_t  = self.conversation_rate
1244
1245         duration2 = max(rate_t, duration * 2)
1246         n = rate * duration2 * rate_n / rate_t
1247
1248         server = 1
1249         client = 2
1250
1251         conversations = []
1252         end = duration2
1253         start = end - duration
1254
1255         while client < n + 2:
1256             start = random.uniform(0, duration2)
1257             c = self.construct_conversation(start,
1258                                             client,
1259                                             server,
1260                                             hard_stop=(duration2 * 5),
1261                                             packet_rate=packet_rate)
1262
1263             c.forget_packets_outside_window(start, end)
1264             c.renormalise_times(start)
1265             if len(c) != 0:
1266                 conversations.append(c)
1267             client += 1
1268
1269         print(("we have %d conversations at rate %f" %
1270                               (len(conversations), rate)), file=sys.stderr)
1271         conversations.sort()
1272         return conversations
1273
1274
1275 IP_PROTOCOLS = {
1276     'dns': '11',
1277     'rpc_netlogon': '06',
1278     'kerberos': '06',      # ratio 16248:258
1279     'smb': '06',
1280     'smb2': '06',
1281     'ldap': '06',
1282     'cldap': '11',
1283     'lsarpc': '06',
1284     'samr': '06',
1285     'dcerpc': '06',
1286     'epm': '06',
1287     'drsuapi': '06',
1288     'browser': '11',
1289     'smb_netlogon': '11',
1290     'srvsvc': '06',
1291     'nbns': '11',
1292 }
1293
1294 OP_DESCRIPTIONS = {
1295     ('browser', '0x01'): 'Host Announcement (0x01)',
1296     ('browser', '0x02'): 'Request Announcement (0x02)',
1297     ('browser', '0x08'): 'Browser Election Request (0x08)',
1298     ('browser', '0x09'): 'Get Backup List Request (0x09)',
1299     ('browser', '0x0c'): 'Domain/Workgroup Announcement (0x0c)',
1300     ('browser', '0x0f'): 'Local Master Announcement (0x0f)',
1301     ('cldap', '3'): 'searchRequest',
1302     ('cldap', '5'): 'searchResDone',
1303     ('dcerpc', '0'): 'Request',
1304     ('dcerpc', '11'): 'Bind',
1305     ('dcerpc', '12'): 'Bind_ack',
1306     ('dcerpc', '13'): 'Bind_nak',
1307     ('dcerpc', '14'): 'Alter_context',
1308     ('dcerpc', '15'): 'Alter_context_resp',
1309     ('dcerpc', '16'): 'AUTH3',
1310     ('dcerpc', '2'): 'Response',
1311     ('dns', '0'): 'query',
1312     ('dns', '1'): 'response',
1313     ('drsuapi', '0'): 'DsBind',
1314     ('drsuapi', '12'): 'DsCrackNames',
1315     ('drsuapi', '13'): 'DsWriteAccountSpn',
1316     ('drsuapi', '1'): 'DsUnbind',
1317     ('drsuapi', '2'): 'DsReplicaSync',
1318     ('drsuapi', '3'): 'DsGetNCChanges',
1319     ('drsuapi', '4'): 'DsReplicaUpdateRefs',
1320     ('epm', '3'): 'Map',
1321     ('kerberos', ''): '',
1322     ('ldap', '0'): 'bindRequest',
1323     ('ldap', '1'): 'bindResponse',
1324     ('ldap', '2'): 'unbindRequest',
1325     ('ldap', '3'): 'searchRequest',
1326     ('ldap', '4'): 'searchResEntry',
1327     ('ldap', '5'): 'searchResDone',
1328     ('ldap', ''): '*** Unknown ***',
1329     ('lsarpc', '14'): 'lsa_LookupNames',
1330     ('lsarpc', '15'): 'lsa_LookupSids',
1331     ('lsarpc', '39'): 'lsa_QueryTrustedDomainInfoBySid',
1332     ('lsarpc', '40'): 'lsa_SetTrustedDomainInfo',
1333     ('lsarpc', '6'): 'lsa_OpenPolicy',
1334     ('lsarpc', '76'): 'lsa_LookupSids3',
1335     ('lsarpc', '77'): 'lsa_LookupNames4',
1336     ('nbns', '0'): 'query',
1337     ('nbns', '1'): 'response',
1338     ('rpc_netlogon', '21'): 'NetrLogonDummyRoutine1',
1339     ('rpc_netlogon', '26'): 'NetrServerAuthenticate3',
1340     ('rpc_netlogon', '29'): 'NetrLogonGetDomainInfo',
1341     ('rpc_netlogon', '30'): 'NetrServerPasswordSet2',
1342     ('rpc_netlogon', '39'): 'NetrLogonSamLogonEx',
1343     ('rpc_netlogon', '40'): 'DsrEnumerateDomainTrusts',
1344     ('rpc_netlogon', '45'): 'NetrLogonSamLogonWithFlags',
1345     ('rpc_netlogon', '4'): 'NetrServerReqChallenge',
1346     ('samr', '0',): 'Connect',
1347     ('samr', '16'): 'GetAliasMembership',
1348     ('samr', '17'): 'LookupNames',
1349     ('samr', '18'): 'LookupRids',
1350     ('samr', '19'): 'OpenGroup',
1351     ('samr', '1'): 'Close',
1352     ('samr', '25'): 'QueryGroupMember',
1353     ('samr', '34'): 'OpenUser',
1354     ('samr', '36'): 'QueryUserInfo',
1355     ('samr', '39'): 'GetGroupsForUser',
1356     ('samr', '3'): 'QuerySecurity',
1357     ('samr', '5'): 'LookupDomain',
1358     ('samr', '64'): 'Connect5',
1359     ('samr', '6'): 'EnumDomains',
1360     ('samr', '7'): 'OpenDomain',
1361     ('samr', '8'): 'QueryDomainInfo',
1362     ('smb', '0x04'): 'Close (0x04)',
1363     ('smb', '0x24'): 'Locking AndX (0x24)',
1364     ('smb', '0x2e'): 'Read AndX (0x2e)',
1365     ('smb', '0x32'): 'Trans2 (0x32)',
1366     ('smb', '0x71'): 'Tree Disconnect (0x71)',
1367     ('smb', '0x72'): 'Negotiate Protocol (0x72)',
1368     ('smb', '0x73'): 'Session Setup AndX (0x73)',
1369     ('smb', '0x74'): 'Logoff AndX (0x74)',
1370     ('smb', '0x75'): 'Tree Connect AndX (0x75)',
1371     ('smb', '0xa2'): 'NT Create AndX (0xa2)',
1372     ('smb2', '0'): 'NegotiateProtocol',
1373     ('smb2', '11'): 'Ioctl',
1374     ('smb2', '14'): 'Find',
1375     ('smb2', '16'): 'GetInfo',
1376     ('smb2', '18'): 'Break',
1377     ('smb2', '1'): 'SessionSetup',
1378     ('smb2', '2'): 'SessionLogoff',
1379     ('smb2', '3'): 'TreeConnect',
1380     ('smb2', '4'): 'TreeDisconnect',
1381     ('smb2', '5'): 'Create',
1382     ('smb2', '6'): 'Close',
1383     ('smb2', '8'): 'Read',
1384     ('smb_netlogon', '0x12'): 'SAM LOGON request from client (0x12)',
1385     ('smb_netlogon', '0x17'): ('SAM Active Directory Response - '
1386                                'user unknown (0x17)'),
1387     ('srvsvc', '16'): 'NetShareGetInfo',
1388     ('srvsvc', '21'): 'NetSrvGetInfo',
1389 }
1390
1391
1392 def expand_short_packet(p, timestamp, src, dest, extra):
1393     protocol, opcode = p.split(':', 1)
1394     desc = OP_DESCRIPTIONS.get((protocol, opcode), '')
1395     ip_protocol = IP_PROTOCOLS.get(protocol, '06')
1396
1397     line = [timestamp, ip_protocol, '', src, dest, protocol, opcode, desc]
1398     line.extend(extra)
1399     return '\t'.join(line)
1400
1401
1402 def replay(conversations,
1403            host=None,
1404            creds=None,
1405            lp=None,
1406            accounts=None,
1407            dns_rate=0,
1408            duration=None,
1409            **kwargs):
1410
1411     context = ReplayContext(server=host,
1412                             creds=creds,
1413                             lp=lp,
1414                             **kwargs)
1415
1416     if len(accounts) < len(conversations):
1417         print(("we have %d accounts but %d conversations" %
1418                (accounts, conversations)), file=sys.stderr)
1419
1420     cstack = list(zip(
1421         sorted(conversations, key=lambda x: x.start_time, reverse=True),
1422         accounts))
1423
1424     # Set the process group so that the calling scripts are not killed
1425     # when the forked child processes are killed.
1426     os.setpgrp()
1427
1428     start = time.time()
1429
1430     if duration is None:
1431         # end 1 second after the last packet of the last conversation
1432         # to start. Conversations other than the last could still be
1433         # going, but we don't care.
1434         duration = cstack[0][0].packets[-1].timestamp + 1.0
1435         print("We will stop after %.1f seconds" % duration,
1436               file=sys.stderr)
1437
1438     end = start + duration
1439
1440     print("Replaying traffic for %u conversations over %d seconds"
1441           % (len(conversations), duration))
1442
1443     children = {}
1444     if dns_rate:
1445         dns_hammer = DnsHammer(dns_rate, duration)
1446         cstack.append((dns_hammer, None))
1447
1448     try:
1449         while True:
1450             # we spawn a batch, wait for finishers, then spawn another
1451             now = time.time()
1452             batch_end = min(now + 2.0, end)
1453             fork_time = 0.0
1454             fork_n = 0
1455             while cstack:
1456                 c, account = cstack.pop()
1457                 if c.start_time + start > batch_end:
1458                     cstack.append((c, account))
1459                     break
1460
1461                 st = time.time()
1462                 pid = c.replay_in_fork_with_delay(start, context, account)
1463                 children[pid] = c
1464                 t = time.time()
1465                 elapsed = t - st
1466                 fork_time += elapsed
1467                 fork_n += 1
1468                 print("forked %s in pid %s (in %fs)" % (c, pid,
1469                                                         elapsed),
1470                       file=sys.stderr)
1471
1472             if fork_n:
1473                 print(("forked %d times in %f seconds (avg %f)" %
1474                        (fork_n, fork_time, fork_time / fork_n)),
1475                       file=sys.stderr)
1476             elif cstack:
1477                 debug(2, "no forks in batch ending %f" % batch_end)
1478
1479             while time.time() < batch_end - 1.0:
1480                 time.sleep(0.01)
1481                 try:
1482                     pid, status = os.waitpid(-1, os.WNOHANG)
1483                 except OSError as e:
1484                     if e.errno != 10:  # no child processes
1485                         raise
1486                     break
1487                 if pid:
1488                     c = children.pop(pid, None)
1489                     print(("process %d finished conversation %s;"
1490                            " %d to go" %
1491                            (pid, c, len(children))), file=sys.stderr)
1492
1493             if time.time() >= end:
1494                 print("time to stop", file=sys.stderr)
1495                 break
1496
1497     except Exception:
1498         print("EXCEPTION in parent", file=sys.stderr)
1499         traceback.print_exc()
1500     finally:
1501         for s in (15, 15, 9):
1502             print(("killing %d children with -%d" %
1503                                  (len(children), s)), file=sys.stderr)
1504             for pid in children:
1505                 try:
1506                     os.kill(pid, s)
1507                 except OSError as e:
1508                     if e.errno != 3:  # don't fail if it has already died
1509                         raise
1510             time.sleep(0.5)
1511             end = time.time() + 1
1512             while children:
1513                 try:
1514                     pid, status = os.waitpid(-1, os.WNOHANG)
1515                 except OSError as e:
1516                     if e.errno != 10:
1517                         raise
1518                 if pid != 0:
1519                     c = children.pop(pid, None)
1520                     print(("kill -%d %d KILLED conversation %s; "
1521                            "%d to go" %
1522                            (s, pid, c, len(children))),
1523                           file=sys.stderr)
1524                 if time.time() >= end:
1525                     break
1526
1527             if not children:
1528                 break
1529             time.sleep(1)
1530
1531         if children:
1532             print("%d children are missing" % len(children),
1533                   file=sys.stderr)
1534
1535         # there may be stragglers that were forked just as ^C was hit
1536         # and don't appear in the list of children. We can get them
1537         # with killpg, but that will also kill us, so this is^H^H would be
1538         # goodbye, except we cheat and pretend to use ^C (SIG_INTERRUPT),
1539         # so as not to have to fuss around writing signal handlers.
1540         try:
1541             os.killpg(0, 2)
1542         except KeyboardInterrupt:
1543             print("ignoring fake ^C", file=sys.stderr)
1544
1545
1546 def openLdb(host, creds, lp):
1547     session = system_session()
1548     ldb = SamDB(url="ldap://%s" % host,
1549                 session_info=session,
1550                 credentials=creds,
1551                 lp=lp)
1552     return ldb
1553
1554
1555 def ou_name(ldb, instance_id):
1556     """Generate an ou name from the instance id"""
1557     return "ou=instance-%d,ou=traffic_replay,%s" % (instance_id,
1558                                                     ldb.domain_dn())
1559
1560
1561 def create_ou(ldb, instance_id):
1562     """Create an ou, all created user and machine accounts will belong to it.
1563
1564     This allows all the created resources to be cleaned up easily.
1565     """
1566     ou = ou_name(ldb, instance_id)
1567     try:
1568         ldb.add({"dn":          ou.split(',', 1)[1],
1569                  "objectclass": "organizationalunit"})
1570     except LdbError as e:
1571         (status, _) = e
1572         # ignore already exists
1573         if status != 68:
1574             raise
1575     try:
1576         ldb.add({"dn":          ou,
1577                  "objectclass": "organizationalunit"})
1578     except LdbError as e:
1579         (status, _) = e
1580         # ignore already exists
1581         if status != 68:
1582             raise
1583     return ou
1584
1585
1586 class ConversationAccounts(object):
1587     """Details of the machine and user accounts associated with a conversation.
1588     """
1589     def __init__(self, netbios_name, machinepass, username, userpass):
1590         self.netbios_name = netbios_name
1591         self.machinepass  = machinepass
1592         self.username     = username
1593         self.userpass     = userpass
1594
1595
1596 def generate_replay_accounts(ldb, instance_id, number, password):
1597     """Generate a series of unique machine and user account names."""
1598
1599     generate_traffic_accounts(ldb, instance_id, number, password)
1600     accounts = []
1601     for i in range(1, number + 1):
1602         netbios_name = "STGM-%d-%d" % (instance_id, i)
1603         username     = "STGU-%d-%d" % (instance_id, i)
1604
1605         account = ConversationAccounts(netbios_name, password, username,
1606                                        password)
1607         accounts.append(account)
1608     return accounts
1609
1610
1611 def generate_traffic_accounts(ldb, instance_id, number, password):
1612     """Create the specified number of user and machine accounts.
1613
1614     As accounts are not explicitly deleted between runs. This function starts
1615     with the last account and iterates backwards stopping either when it
1616     finds an already existing account or it has generated all the required
1617     accounts.
1618     """
1619     print(("Generating machine and conversation accounts, "
1620            "as required for %d conversations" % number),
1621           file=sys.stderr)
1622     added = 0
1623     for i in range(number, 0, -1):
1624         try:
1625             netbios_name = "STGM-%d-%d" % (instance_id, i)
1626             create_machine_account(ldb, instance_id, netbios_name, password)
1627             added += 1
1628         except LdbError as e:
1629             (status, _) = e
1630             if status == 68:
1631                 break
1632             else:
1633                 raise
1634     if added > 0:
1635         print("Added %d new machine accounts" % added,
1636               file=sys.stderr)
1637
1638     added = 0
1639     for i in range(number, 0, -1):
1640         try:
1641             username = "STGU-%d-%d" % (instance_id, i)
1642             create_user_account(ldb, instance_id, username, password)
1643             added += 1
1644         except LdbError as e:
1645             (status, _) = e
1646             if status == 68:
1647                 break
1648             else:
1649                 raise
1650
1651     if added > 0:
1652         print("Added %d new user accounts" % added,
1653               file=sys.stderr)
1654
1655
1656 def create_machine_account(ldb, instance_id, netbios_name, machinepass):
1657     """Create a machine account via ldap."""
1658
1659     ou = ou_name(ldb, instance_id)
1660     dn = "cn=%s,%s" % (netbios_name, ou)
1661     utf16pw = unicode(
1662         '"' + machinepass.encode('utf-8') + '"', 'utf-8'
1663     ).encode('utf-16-le')
1664     start = time.time()
1665     ldb.add({
1666         "dn": dn,
1667         "objectclass": "computer",
1668         "sAMAccountName": "%s$" % netbios_name,
1669         "userAccountControl":
1670             str(UF_TRUSTED_FOR_DELEGATION | UF_SERVER_TRUST_ACCOUNT),
1671         "unicodePwd": utf16pw})
1672     end = time.time()
1673     duration = end - start
1674     print("%f\t0\tcreate\tmachine\t%f\tTrue\t" % (end, duration))
1675
1676
1677 def create_user_account(ldb, instance_id, username, userpass):
1678     """Create a user account via ldap."""
1679     ou = ou_name(ldb, instance_id)
1680     user_dn = "cn=%s,%s" % (username, ou)
1681     utf16pw = unicode(
1682         '"' + userpass.encode('utf-8') + '"', 'utf-8'
1683     ).encode('utf-16-le')
1684     start = time.time()
1685     ldb.add({
1686         "dn": user_dn,
1687         "objectclass": "user",
1688         "sAMAccountName": username,
1689         "userAccountControl": str(UF_NORMAL_ACCOUNT),
1690         "unicodePwd": utf16pw
1691     })
1692
1693     # grant user write permission to do things like write account SPN
1694     sdutils = sd_utils.SDUtils(ldb)
1695     sdutils.dacl_add_ace(user_dn,  "(A;;WP;;;PS)")
1696
1697     end = time.time()
1698     duration = end - start
1699     print("%f\t0\tcreate\tuser\t%f\tTrue\t" % (end, duration))
1700
1701
1702 def create_group(ldb, instance_id, name):
1703     """Create a group via ldap."""
1704
1705     ou = ou_name(ldb, instance_id)
1706     dn = "cn=%s,%s" % (name, ou)
1707     start = time.time()
1708     ldb.add({
1709         "dn": dn,
1710         "objectclass": "group",
1711     })
1712     end = time.time()
1713     duration = end - start
1714     print("%f\t0\tcreate\tgroup\t%f\tTrue\t" % (end, duration))
1715
1716
1717 def user_name(instance_id, i):
1718     """Generate a user name based in the instance id"""
1719     return "STGU-%d-%d" % (instance_id, i)
1720
1721
1722 def generate_users(ldb, instance_id, number, password):
1723     """Add users to the server"""
1724     users = 0
1725     for i in range(number, 0, -1):
1726         try:
1727             username = user_name(instance_id, i)
1728             create_user_account(ldb, instance_id, username, password)
1729             users += 1
1730         except LdbError as e:
1731             (status, _) = e
1732             # Stop if entry exists
1733             if status == 68:
1734                 break
1735             else:
1736                 raise
1737
1738     return users
1739
1740
1741 def group_name(instance_id, i):
1742     """Generate a group name from instance id."""
1743     return "STGG-%d-%d" % (instance_id, i)
1744
1745
1746 def generate_groups(ldb, instance_id, number):
1747     """Create the required number of groups on the server."""
1748     groups = 0
1749     for i in range(number, 0, -1):
1750         try:
1751             name = group_name(instance_id, i)
1752             create_group(ldb, instance_id, name)
1753             groups += 1
1754         except LdbError as e:
1755             (status, _) = e
1756             # Stop if entry exists
1757             if status == 68:
1758                 break
1759             else:
1760                 raise
1761     return groups
1762
1763
1764 def clean_up_accounts(ldb, instance_id):
1765     """Remove the created accounts and groups from the server."""
1766     ou = ou_name(ldb, instance_id)
1767     try:
1768         ldb.delete(ou, ["tree_delete:1"])
1769     except LdbError as e:
1770         (status, _) = e
1771         # ignore does not exist
1772         if status != 32:
1773             raise
1774
1775
1776 def generate_users_and_groups(ldb, instance_id, password,
1777                               number_of_users, number_of_groups,
1778                               group_memberships):
1779     """Generate the required users and groups, allocating the users to
1780        those groups."""
1781     assignments = []
1782     groups_added  = 0
1783
1784     create_ou(ldb, instance_id)
1785
1786     print("Generating dummy user accounts", file=sys.stderr)
1787     users_added = generate_users(ldb, instance_id, number_of_users, password)
1788
1789     if number_of_groups > 0:
1790         print("Generating dummy groups", file=sys.stderr)
1791         groups_added = generate_groups(ldb, instance_id, number_of_groups)
1792
1793     if group_memberships > 0:
1794         print("Assigning users to groups", file=sys.stderr)
1795         assignments = assign_groups(number_of_groups,
1796                                     groups_added,
1797                                     number_of_users,
1798                                     users_added,
1799                                     group_memberships)
1800         print("Adding users to groups", file=sys.stderr)
1801         add_users_to_groups(ldb, instance_id, assignments)
1802
1803     if (groups_added > 0 and users_added == 0 and
1804        number_of_groups != groups_added):
1805         print("Warning: the added groups will contain no members",
1806               file=sys.stderr)
1807
1808     print(("Added %d users, %d groups and %d group memberships" %
1809            (users_added, groups_added, len(assignments))),
1810           file=sys.stderr)
1811
1812
1813 def assign_groups(number_of_groups,
1814                   groups_added,
1815                   number_of_users,
1816                   users_added,
1817                   group_memberships):
1818     """Allocate users to groups.
1819
1820     The intention is to have a few users that belong to most groups, while
1821     the majority of users belong to a few groups.
1822
1823     A few groups will contain most users, with the remaining only having a
1824     few users.
1825     """
1826
1827     def generate_user_distribution(n):
1828         """Probability distribution of a user belonging to a group.
1829         """
1830         dist = []
1831         for x in range(1, n + 1):
1832             p = 1 / (x + 0.001)
1833             dist.append(p)
1834         return dist
1835
1836     def generate_group_distribution(n):
1837         """Probability distribution of a group containing a user."""
1838         dist = []
1839         for x in range(1, n + 1):
1840             p = 1 / (x**1.3)
1841             dist.append(p)
1842         return dist
1843
1844     assignments = set()
1845     if group_memberships <= 0:
1846         return assignments
1847
1848     group_dist = generate_group_distribution(number_of_groups)
1849     user_dist  = generate_user_distribution(number_of_users)
1850
1851     # Calculate the number of group menberships required
1852     group_memberships = math.ceil(
1853         float(group_memberships) *
1854         (float(users_added) / float(number_of_users)))
1855
1856     existing_users  = number_of_users  - users_added  - 1
1857     existing_groups = number_of_groups - groups_added - 1
1858     while len(assignments) < group_memberships:
1859         user        = random.randint(0, number_of_users - 1)
1860         group       = random.randint(0, number_of_groups - 1)
1861         probability = group_dist[group] * user_dist[user]
1862
1863         if ((random.random() < probability * 10000) and
1864            (group > existing_groups or user > existing_users)):
1865             # the + 1 converts the array index to the corresponding
1866             # group or user number
1867             assignments.add(((user + 1), (group + 1)))
1868
1869     return assignments
1870
1871
1872 def add_users_to_groups(db, instance_id, assignments):
1873     """Add users to their assigned groups.
1874
1875     Takes the list of (group,user) tuples generated by assign_groups and
1876     assign the users to their specified groups."""
1877
1878     ou = ou_name(db, instance_id)
1879
1880     def build_dn(name):
1881         return("cn=%s,%s" % (name, ou))
1882
1883     for (user, group) in assignments:
1884         user_dn  = build_dn(user_name(instance_id, user))
1885         group_dn = build_dn(group_name(instance_id, group))
1886
1887         m = ldb.Message()
1888         m.dn = ldb.Dn(db, group_dn)
1889         m["member"] = ldb.MessageElement(user_dn, ldb.FLAG_MOD_ADD, "member")
1890         start = time.time()
1891         db.modify(m)
1892         end = time.time()
1893         duration = end - start
1894         print("%f\t0\tadd\tuser\t%f\tTrue\t" % (end, duration))
1895
1896
1897 def generate_stats(statsdir, timing_file):
1898     """Generate and print the summary stats for a run."""
1899     first      = sys.float_info.max
1900     last       = 0
1901     successful = 0
1902     failed     = 0
1903     latencies  = {}
1904     failures   = {}
1905     unique_converations = set()
1906     conversations = 0
1907
1908     if timing_file is not None:
1909         tw = timing_file.write
1910     else:
1911         def tw(x):
1912             pass
1913
1914     tw("time\tconv\tprotocol\ttype\tduration\tsuccessful\terror\n")
1915
1916     for filename in os.listdir(statsdir):
1917         path = os.path.join(statsdir, filename)
1918         with open(path, 'r') as f:
1919             for line in f:
1920                 try:
1921                     fields       = line.rstrip('\n').split('\t')
1922                     conversation = fields[1]
1923                     protocol     = fields[2]
1924                     packet_type  = fields[3]
1925                     latency      = float(fields[4])
1926                     first        = min(float(fields[0]) - latency, first)
1927                     last         = max(float(fields[0]), last)
1928
1929                     if protocol not in latencies:
1930                         latencies[protocol] = {}
1931                     if packet_type not in latencies[protocol]:
1932                         latencies[protocol][packet_type] = []
1933
1934                     latencies[protocol][packet_type].append(latency)
1935
1936                     if protocol not in failures:
1937                         failures[protocol] = {}
1938                     if packet_type not in failures[protocol]:
1939                         failures[protocol][packet_type] = 0
1940
1941                     if fields[5] == 'True':
1942                         successful += 1
1943                     else:
1944                         failed += 1
1945                         failures[protocol][packet_type] += 1
1946
1947                     if conversation not in unique_converations:
1948                         unique_converations.add(conversation)
1949                         conversations += 1
1950
1951                     tw(line)
1952                 except (ValueError, IndexError):
1953                     # not a valid line print and ignore
1954                     print(line, file=sys.stderr)
1955                     pass
1956     duration = last - first
1957     if successful == 0:
1958         success_rate = 0
1959     else:
1960         success_rate = successful / duration
1961     if failed == 0:
1962         failure_rate = 0
1963     else:
1964         failure_rate = failed / duration
1965
1966     # print the stats in more human-readable format when stdout is going to the
1967     # console (as opposed to being redirected to a file)
1968     if sys.stdout.isatty():
1969         print("Total conversations:   %10d" % conversations)
1970         print("Successful operations: %10d (%.3f per second)"
1971               % (successful, success_rate))
1972         print("Failed operations:     %10d (%.3f per second)"
1973               % (failed, failure_rate))
1974     else:
1975         print("(%d, %d, %d, %.3f, %.3f)" %
1976               (conversations, successful, failed, success_rate, failure_rate))
1977
1978     if sys.stdout.isatty():
1979         print("Protocol    Op Code  Description                               "
1980               " Count       Failed         Mean       Median          "
1981               "95%        Range          Max")
1982     else:
1983         print("proto\top_code\tdesc\tcount\tfailed\tmean\tmedian\t95%\trange"
1984               "\tmax")
1985     protocols = sorted(latencies.keys())
1986     for protocol in protocols:
1987         packet_types = sorted(latencies[protocol], key=opcode_key)
1988         for packet_type in packet_types:
1989             values     = latencies[protocol][packet_type]
1990             values     = sorted(values)
1991             count      = len(values)
1992             failed     = failures[protocol][packet_type]
1993             mean       = sum(values) / count
1994             median     = calc_percentile(values, 0.50)
1995             percentile = calc_percentile(values, 0.95)
1996             rng        = values[-1] - values[0]
1997             maxv       = values[-1]
1998             desc       = OP_DESCRIPTIONS.get((protocol, packet_type), '')
1999             if sys.stdout.isatty:
2000                 print("%-12s   %4s  %-35s %12d %12d %12.6f "
2001                       "%12.6f %12.6f %12.6f %12.6f"
2002                       % (protocol,
2003                          packet_type,
2004                          desc,
2005                          count,
2006                          failed,
2007                          mean,
2008                          median,
2009                          percentile,
2010                          rng,
2011                          maxv))
2012             else:
2013                 print("%s\t%s\t%s\t%d\t%d\t%f\t%f\t%f\t%f\t%f"
2014                       % (protocol,
2015                          packet_type,
2016                          desc,
2017                          count,
2018                          failed,
2019                          mean,
2020                          median,
2021                          percentile,
2022                          rng,
2023                          maxv))
2024
2025
2026 def opcode_key(v):
2027     """Sort key for the operation code to ensure that it sorts numerically"""
2028     try:
2029         return "%03d" % int(v)
2030     except:
2031         return v
2032
2033
2034 def calc_percentile(values, percentile):
2035     """Calculate the specified percentile from the list of values.
2036
2037     Assumes the list is sorted in ascending order.
2038     """
2039
2040     if not values:
2041         return 0
2042     k = (len(values) - 1) * percentile
2043     f = math.floor(k)
2044     c = math.ceil(k)
2045     if f == c:
2046         return values[int(k)]
2047     d0 = values[int(f)] * (c - k)
2048     d1 = values[int(c)] * (k - f)
2049     return d0 + d1
2050
2051
2052 def mk_masked_dir(*path):
2053     """In a testenv we end up with 0777 diectories that look an alarming
2054     green colour with ls. Use umask to avoid that."""
2055     d = os.path.join(*path)
2056     mask = os.umask(0o077)
2057     os.mkdir(d)
2058     os.umask(mask)
2059     return d