source4/dsdb/tests/python/ad_dc_search_performance.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 from __future__ import print_function
   4
   5 import optparse
   6 import sys
   7 sys.path.insert(0, 'bin/python')
   8
   9 import os
  10 import samba
  11 import samba.getopt as options
  12 import random
  13 import tempfile
  14 import shutil
  15 import time
  16 import itertools
  17
  18 from samba.netcmd.main import cmd_sambatool
  19
  20 # We try to use the test infrastructure of Samba 4.3+, but if it
  21 # doesn't work, we are probably in a back-ported patch and trying to
  22 # run on 4.1 or something.
  23 #
  24 # Don't copy this horror into ordinary tests -- it is special for
  25 # performance tests that want to apply to old versions.
  26 try:
  27     from samba.tests.subunitrun import SubunitOptions, TestProgram
  28     ANCIENT_SAMBA = False
  29 except ImportError:
  30     ANCIENT_SAMBA = True
  31     samba.ensure_external_module("testtools", "testtools")
  32     samba.ensure_external_module("subunit", "subunit/python")
  33     from subunit.run import SubunitTestRunner
  34     import unittest
  35
  36 from samba.samdb import SamDB
  37 from samba.auth import system_session
  38 from ldb import Message, MessageElement, Dn, LdbError
  39 from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
  40 from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
  41
  42 parser = optparse.OptionParser("ad_dc_performance.py [options] <host>")
  43 sambaopts = options.SambaOptions(parser)
  44 parser.add_option_group(sambaopts)
  45 parser.add_option_group(options.VersionOptions(parser))
  46
  47 if not ANCIENT_SAMBA:
  48     subunitopts = SubunitOptions(parser)
  49     parser.add_option_group(subunitopts)
  50
  51 # use command line creds if available
  52 credopts = options.CredentialsOptions(parser)
  53 parser.add_option_group(credopts)
  54 opts, args = parser.parse_args()
  55
  56
  57 if len(args) < 1:
  58     parser.print_usage()
  59     sys.exit(1)
  60
  61 host = args[0]
  62
  63 lp = sambaopts.get_loadparm()
  64 creds = credopts.get_credentials(lp)
  65
  66 random.seed(1)
  67
  68
  69 class PerfTestException(Exception):
  70     pass
  71
  72
  73 BATCH_SIZE = 1000
  74 N_GROUPS = 5
  75
  76
  77 class GlobalState(object):
  78     next_user_id = 0
  79     n_groups = 0
  80     next_linked_user = 0
  81     next_relinked_user = 0
  82     next_linked_user_3 = 0
  83     next_removed_link_0 = 0
  84
  85
  86 class UserTests(samba.tests.TestCase):
  87
  88     def add_if_possible(self, *args, **kwargs):
  89         """In these tests sometimes things are left in the database
  90         deliberately, so we don't worry if we fail to add them a second
  91         time."""
  92         try:
  93             self.ldb.add(*args, **kwargs)
  94         except LdbError:
  95             pass
  96
  97     def setUp(self):
  98         super(UserTests, self).setUp()
  99         self.state = GlobalState  # the class itself, not an instance
 100         self.lp = lp
 101         self.ldb = SamDB(host, credentials=creds,
 102                          session_info=system_session(lp), lp=lp)
 103         self.base_dn = self.ldb.domain_dn()
 104         self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn)
 105         self.ou_users = "OU=users,%s" % self.ou
 106         self.ou_groups = "OU=groups,%s" % self.ou
 107         self.ou_computers = "OU=computers,%s" % self.ou
 108
 109         for dn in (self.ou, self.ou_users, self.ou_groups,
 110                    self.ou_computers):
 111             self.add_if_possible({
 112                 "dn": dn,
 113                 "objectclass": "organizationalUnit"})
 114
 115     def tearDown(self):
 116         super(UserTests, self).tearDown()
 117
 118     def test_00_00_do_nothing(self):
 119         # this gives us an idea of the overhead
 120         pass
 121
 122     def _prepare_n_groups(self, n):
 123         self.state.n_groups = n
 124         for i in range(n):
 125             self.add_if_possible({
 126                 "dn": "cn=g%d,%s" % (i, self.ou_groups),
 127                 "objectclass": "group"})
 128
 129     def _add_users(self, start, end):
 130         for i in range(start, end):
 131             self.ldb.add({
 132                 "dn": "cn=u%d,%s" % (i, self.ou_users),
 133                 "objectclass": "user"})
 134
 135     def _add_users_ldif(self, start, end):
 136         lines = []
 137         for i in range(start, end):
 138             lines.append("dn: cn=u%d,%s" % (i, self.ou_users))
 139             lines.append("objectclass: user")
 140             lines.append("")
 141         self.ldb.add_ldif('\n'.join(lines))
 142
 143     def _test_unindexed_search(self):
 144         expressions = [
 145             ('(&(objectclass=user)(description='
 146              'Built-in account for adminstering the computer/domain))'),
 147             '(description=Built-in account for adminstering the computer/domain)',
 148             '(objectCategory=*)',
 149             '(samaccountname=Administrator*)'
 150         ]
 151         for expression in expressions:
 152             t = time.time()
 153             for i in range(50):
 154                 self.ldb.search(self.ou,
 155                                 expression=expression,
 156                                 scope=SCOPE_SUBTREE,
 157                                 attrs=['cn'])
 158             print('%d %s took %s' % (i, expression,
 159                                      time.time() - t),
 160                   file=sys.stderr)
 161
 162     def _test_indexed_search(self):
 163         expressions = ['(objectclass=group)',
 164                        '(samaccountname=Administrator)'
 165                        ]
 166         for expression in expressions:
 167             t = time.time()
 168             for i in range(10000):
 169                 self.ldb.search(self.ou,
 170                                 expression=expression,
 171                                 scope=SCOPE_SUBTREE,
 172                                 attrs=['cn'])
 173             print('%d runs %s took %s' % (i, expression,
 174                                           time.time() - t),
 175                   file=sys.stderr)
 176
 177     def _test_complex_search(self):
 178         classes = ['samaccountname', 'objectCategory', 'dn', 'member']
 179         values = ['*', '*t*', 'g*', 'user']
 180         comparators = ['=', '<=', '>=']  # '~=' causes error
 181         maybe_not = ['!(', '']
 182         joiners = ['&', '|']
 183
 184         # The number of permuations is 18432, which is not huge but
 185         # would take hours to search. So we take a sample.
 186         all_permutations = list(itertools.product(joiners,
 187                                                   classes, classes,
 188                                                   values, values,
 189                                                   comparators, comparators,
 190                                                   maybe_not, maybe_not))
 191         random.seed(1)
 192
 193         for (j, c1, c2, v1, v2,
 194              o1, o2, n1, n2) in random.sample(all_permutations, 100):
 195             expression = ''.join(['(', j,
 196                                   '(', n1, c1, o1, v1,
 197                                   '))' if n1 else ')',
 198                                   '(', n2, c2, o2, v2,
 199                                   '))' if n2 else ')',
 200                                   ')'])
 201             print(expression)
 202             self.ldb.search(self.ou,
 203                             expression=expression,
 204                             scope=SCOPE_SUBTREE,
 205                             attrs=['cn'])
 206
 207     def _test_member_search(self, rounds=10):
 208         expressions = []
 209         for d in range(50):
 210             expressions.append('(member=cn=u%d,%s)' % (d + 500, self.ou_users))
 211             expressions.append('(member=u%d*)' % (d + 700,))
 212         for i in range(N_GROUPS):
 213             expressions.append('(memberOf=cn=g%d,%s)' % (i, self.ou_groups))
 214             expressions.append('(memberOf=cn=g%d*)' % (i,))
 215             expressions.append('(memberOf=cn=*%s*)' % self.ou_groups)
 216
 217         for expression in expressions:
 218             t = time.time()
 219             for i in range(rounds):
 220                 self.ldb.search(self.ou,
 221                                 expression=expression,
 222                                 scope=SCOPE_SUBTREE,
 223                                 attrs=['cn'])
 224             print('%d runs %s took %s' % (i, expression,
 225                                           time.time() - t),
 226                   file=sys.stderr)
 227
 228     def _test_add_many_users(self, n=BATCH_SIZE):
 229         s = self.state.next_user_id
 230         e = s + n
 231         self._add_users(s, e)
 232         self.state.next_user_id = e
 233
 234     def _test_add_many_users_ldif(self, n=BATCH_SIZE):
 235         s = self.state.next_user_id
 236         e = s + n
 237         self._add_users_ldif(s, e)
 238         self.state.next_user_id = e
 239
 240     def _link_user_and_group(self, u, g):
 241         m = Message()
 242         m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups))
 243         m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users),
 244                                      FLAG_MOD_ADD, "member")
 245         self.ldb.modify(m)
 246
 247     def _test_link_many_users(self, n=BATCH_SIZE):
 248         self._prepare_n_groups(N_GROUPS)
 249         s = self.state.next_linked_user
 250         e = s + n
 251         for i in range(s, e):
 252             # put everyone in group 0, and one other group
 253             g = i % (N_GROUPS - 1) + 1
 254             self._link_user_and_group(i, g)
 255             self._link_user_and_group(i, 0)
 256         self.state.next_linked_user = e
 257
 258     test_00_01_adding_users_1000 = _test_add_many_users
 259
 260     test_00_10_complex_search_1k_users = _test_complex_search
 261     test_00_11_unindexed_search_1k_users = _test_unindexed_search
 262     test_00_12_indexed_search_1k_users = _test_indexed_search
 263     test_00_13_member_search_1k_users = _test_member_search
 264
 265     test_01_02_adding_users_2000_ldif = _test_add_many_users_ldif
 266     test_01_03_adding_users_3000 = _test_add_many_users
 267
 268     test_01_10_complex_search_3k_users = _test_complex_search
 269     test_01_11_unindexed_search_3k_users = _test_unindexed_search
 270     test_01_12_indexed_search_3k_users = _test_indexed_search
 271
 272     def test_01_13_member_search_3k_users(self):
 273         self._test_member_search(rounds=5)
 274
 275     test_02_01_link_users_1000 = _test_link_many_users
 276     test_02_02_link_users_2000 = _test_link_many_users
 277     test_02_03_link_users_3000 = _test_link_many_users
 278
 279     test_03_10_complex_search_linked_users = _test_complex_search
 280     test_03_11_unindexed_search_linked_users = _test_unindexed_search
 281     test_03_12_indexed_search_linked_users = _test_indexed_search
 282
 283     def test_03_13_member_search_linked_users(self):
 284         self._test_member_search(rounds=2)
 285
 286 if "://" not in host:
 287     if os.path.isfile(host):
 288         host = "tdb://%s" % host
 289     else:
 290         host = "ldap://%s" % host
 291
 292
 293 if ANCIENT_SAMBA:
 294     runner = SubunitTestRunner()
 295     if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
 296         sys.exit(1)
 297     sys.exit(0)
 298 else:
 299     TestProgram(module=__name__, opts=subunitopts)