source4/dsdb/tests/python/ad_dc_search_performance.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 import optparse
   5 import sys
   6 sys.path.insert(0, 'bin/python')
   7
   8 import os
   9 import samba
  10 import samba.getopt as options
  11 import random
  12 import tempfile
  13 import shutil
  14 import time
  15 import itertools
  16
  17 from samba.netcmd.main import cmd_sambatool
  18
  19 # We try to use the test infrastructure of Samba 4.3+, but if it
  20 # doesn't work, we are probably in a back-ported patch and trying to
  21 # run on 4.1 or something.
  22 #
  23 # Don't copy this horror into ordinary tests -- it is special for
  24 # performance tests that want to apply to old versions.
  25 try:
  26     from samba.tests.subunitrun import SubunitOptions, TestProgram
  27     ANCIENT_SAMBA = False
  28 except ImportError:
  29     ANCIENT_SAMBA = True
  30     samba.ensure_external_module("testtools", "testtools")
  31     samba.ensure_external_module("subunit", "subunit/python")
  32     from subunit.run import SubunitTestRunner
  33     import unittest
  34
  35 from samba.samdb import SamDB
  36 from samba.auth import system_session
  37 from ldb import Message, MessageElement, Dn, LdbError
  38 from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
  39 from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
  40
  41 parser = optparse.OptionParser("ad_dc_performance.py [options] <host>")
  42 sambaopts = options.SambaOptions(parser)
  43 parser.add_option_group(sambaopts)
  44 parser.add_option_group(options.VersionOptions(parser))
  45
  46 if not ANCIENT_SAMBA:
  47     subunitopts = SubunitOptions(parser)
  48     parser.add_option_group(subunitopts)
  49
  50 # use command line creds if available
  51 credopts = options.CredentialsOptions(parser)
  52 parser.add_option_group(credopts)
  53 opts, args = parser.parse_args()
  54
  55
  56 if len(args) < 1:
  57     parser.print_usage()
  58     sys.exit(1)
  59
  60 host = args[0]
  61
  62 lp = sambaopts.get_loadparm()
  63 creds = credopts.get_credentials(lp)
  64
  65 random.seed(1)
  66
  67
  68 class PerfTestException(Exception):
  69     pass
  70
  71
  72 BATCH_SIZE = 1000
  73 N_GROUPS = 5
  74
  75
  76 class GlobalState(object):
  77     next_user_id = 0
  78     n_groups = 0
  79     next_linked_user = 0
  80     next_relinked_user = 0
  81     next_linked_user_3 = 0
  82     next_removed_link_0 = 0
  83
  84
  85 class UserTests(samba.tests.TestCase):
  86
  87     def add_if_possible(self, *args, **kwargs):
  88         """In these tests sometimes things are left in the database
  89         deliberately, so we don't worry if we fail to add them a second
  90         time."""
  91         try:
  92             self.ldb.add(*args, **kwargs)
  93         except LdbError:
  94             pass
  95
  96     def setUp(self):
  97         super(UserTests, self).setUp()
  98         self.state = GlobalState  # the class itself, not an instance
  99         self.lp = lp
 100         self.ldb = SamDB(host, credentials=creds,
 101                          session_info=system_session(lp), lp=lp)
 102         self.base_dn = self.ldb.domain_dn()
 103         self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn)
 104         self.ou_users = "OU=users,%s" % self.ou
 105         self.ou_groups = "OU=groups,%s" % self.ou
 106         self.ou_computers = "OU=computers,%s" % self.ou
 107
 108         for dn in (self.ou, self.ou_users, self.ou_groups,
 109                    self.ou_computers):
 110             self.add_if_possible({
 111                 "dn": dn,
 112                 "objectclass": "organizationalUnit"})
 113
 114     def tearDown(self):
 115         super(UserTests, self).tearDown()
 116
 117     def test_00_00_do_nothing(self):
 118         # this gives us an idea of the overhead
 119         pass
 120
 121     def _prepare_n_groups(self, n):
 122         self.state.n_groups = n
 123         for i in range(n):
 124             self.add_if_possible({
 125                 "dn": "cn=g%d,%s" % (i, self.ou_groups),
 126                 "objectclass": "group"})
 127
 128     def _add_users(self, start, end):
 129         for i in range(start, end):
 130             self.ldb.add({
 131                 "dn": "cn=u%d,%s" % (i, self.ou_users),
 132                 "objectclass": "user"})
 133
 134     def _add_users_ldif(self, start, end):
 135         lines = []
 136         for i in range(start, end):
 137             lines.append("dn: cn=u%d,%s" % (i, self.ou_users))
 138             lines.append("objectclass: user")
 139             lines.append("")
 140         self.ldb.add_ldif('\n'.join(lines))
 141
 142     def _test_unindexed_search(self):
 143         expressions = [
 144             ('(&(objectclass=user)(description='
 145              'Built-in account for adminstering the computer/domain))'),
 146             '(description=Built-in account for adminstering the computer/domain)',
 147             '(objectCategory=*)',
 148             '(samaccountname=Administrator*)'
 149         ]
 150         for expression in expressions:
 151             t = time.time()
 152             for i in range(50):
 153                 self.ldb.search(self.ou,
 154                                 expression=expression,
 155                                 scope=SCOPE_SUBTREE,
 156                                 attrs=['cn'])
 157             print('%d %s took %s' % (i, expression,
 158                                      time.time() - t),
 159                   file=sys.stderr)
 160
 161     def _test_indexed_search(self):
 162         expressions = ['(objectclass=group)',
 163                        '(samaccountname=Administrator)'
 164                        ]
 165         for expression in expressions:
 166             t = time.time()
 167             for i in range(10000):
 168                 self.ldb.search(self.ou,
 169                                 expression=expression,
 170                                 scope=SCOPE_SUBTREE,
 171                                 attrs=['cn'])
 172             print('%d runs %s took %s' % (i, expression,
 173                                           time.time() - t),
 174                   file=sys.stderr)
 175
 176     def _test_complex_search(self):
 177         classes = ['samaccountname', 'objectCategory', 'dn', 'member']
 178         values = ['*', '*t*', 'g*', 'user']
 179         comparators = ['=', '<=', '>=']  # '~=' causes error
 180         maybe_not = ['!(', '']
 181         joiners = ['&', '|']
 182
 183         # The number of permuations is 18432, which is not huge but
 184         # would take hours to search. So we take a sample.
 185         all_permutations = list(itertools.product(joiners,
 186                                                   classes, classes,
 187                                                   values, values,
 188                                                   comparators, comparators,
 189                                                   maybe_not, maybe_not))
 190         random.seed(1)
 191
 192         for (j, c1, c2, v1, v2,
 193              o1, o2, n1, n2) in random.sample(all_permutations, 100):
 194             expression = ''.join(['(', j,
 195                                   '(', n1, c1, o1, v1,
 196                                   '))' if n1 else ')',
 197                                   '(', n2, c2, o2, v2,
 198                                   '))' if n2 else ')',
 199                                   ')'])
 200             print(expression)
 201             self.ldb.search(self.ou,
 202                             expression=expression,
 203                             scope=SCOPE_SUBTREE,
 204                             attrs=['cn'])
 205
 206     def _test_member_search(self, rounds=10):
 207         expressions = []
 208         for d in range(50):
 209             expressions.append('(member=cn=u%d,%s)' % (d + 500, self.ou_users))
 210             expressions.append('(member=u%d*)' % (d + 700,))
 211         for i in range(N_GROUPS):
 212             expressions.append('(memberOf=cn=g%d,%s)' % (i, self.ou_groups))
 213             expressions.append('(memberOf=cn=g%d*)' % (i,))
 214             expressions.append('(memberOf=cn=*%s*)' % self.ou_groups)
 215
 216         for expression in expressions:
 217             t = time.time()
 218             for i in range(rounds):
 219                 self.ldb.search(self.ou,
 220                                 expression=expression,
 221                                 scope=SCOPE_SUBTREE,
 222                                 attrs=['cn'])
 223             print('%d runs %s took %s' % (i, expression,
 224                                           time.time() - t),
 225                   file=sys.stderr)
 226
 227     def _test_add_many_users(self, n=BATCH_SIZE):
 228         s = self.state.next_user_id
 229         e = s + n
 230         self._add_users(s, e)
 231         self.state.next_user_id = e
 232
 233     def _test_add_many_users_ldif(self, n=BATCH_SIZE):
 234         s = self.state.next_user_id
 235         e = s + n
 236         self._add_users_ldif(s, e)
 237         self.state.next_user_id = e
 238
 239     def _link_user_and_group(self, u, g):
 240         m = Message()
 241         m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups))
 242         m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users),
 243                                      FLAG_MOD_ADD, "member")
 244         self.ldb.modify(m)
 245
 246     def _test_link_many_users(self, n=BATCH_SIZE):
 247         self._prepare_n_groups(N_GROUPS)
 248         s = self.state.next_linked_user
 249         e = s + n
 250         for i in range(s, e):
 251             # put everyone in group 0, and one other group
 252             g = i % (N_GROUPS - 1) + 1
 253             self._link_user_and_group(i, g)
 254             self._link_user_and_group(i, 0)
 255         self.state.next_linked_user = e
 256
 257     test_00_01_adding_users_1000 = _test_add_many_users
 258
 259     test_00_10_complex_search_1k_users = _test_complex_search
 260     test_00_11_unindexed_search_1k_users = _test_unindexed_search
 261     test_00_12_indexed_search_1k_users = _test_indexed_search
 262     test_00_13_member_search_1k_users = _test_member_search
 263
 264     test_01_02_adding_users_2000_ldif = _test_add_many_users_ldif
 265     test_01_03_adding_users_3000 = _test_add_many_users
 266
 267     test_01_10_complex_search_3k_users = _test_complex_search
 268     test_01_11_unindexed_search_3k_users = _test_unindexed_search
 269     test_01_12_indexed_search_3k_users = _test_indexed_search
 270
 271     def test_01_13_member_search_3k_users(self):
 272         self._test_member_search(rounds=5)
 273
 274     test_02_01_link_users_1000 = _test_link_many_users
 275     test_02_02_link_users_2000 = _test_link_many_users
 276     test_02_03_link_users_3000 = _test_link_many_users
 277
 278     test_03_10_complex_search_linked_users = _test_complex_search
 279     test_03_11_unindexed_search_linked_users = _test_unindexed_search
 280     test_03_12_indexed_search_linked_users = _test_indexed_search
 281
 282     def test_03_13_member_search_linked_users(self):
 283         self._test_member_search(rounds=2)
 284
 285
 286 if "://" not in host:
 287     if os.path.isfile(host):
 288         host = "tdb://%s" % host
 289     else:
 290         host = "ldap://%s" % host
 291
 292
 293 if ANCIENT_SAMBA:
 294     runner = SubunitTestRunner()
 295     if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
 296         sys.exit(1)
 297     sys.exit(0)
 298 else:
 299     TestProgram(module=__name__, opts=subunitopts)