source4/dsdb/tests/python/ad_dc_performance.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 from __future__ import print_function
   4
   5 import optparse
   6 import sys
   7 sys.path.insert(0, 'bin/python')
   8
   9 import os
  10 import samba
  11 import samba.getopt as options
  12 import random
  13 import tempfile
  14 import shutil
  15 import time
  16
  17 from samba.netcmd.main import cmd_sambatool
  18
  19 # We try to use the test infrastructure of Samba 4.3+, but if it
  20 # doesn't work, we are probably in a back-ported patch and trying to
  21 # run on 4.1 or something.
  22 #
  23 # Don't copy this horror into ordinary tests -- it is special for
  24 # performance tests that want to apply to old versions.
  25 try:
  26     from samba.tests.subunitrun import SubunitOptions, TestProgram
  27     ANCIENT_SAMBA = False
  28 except ImportError:
  29     ANCIENT_SAMBA = True
  30     samba.ensure_external_module("testtools", "testtools")
  31     samba.ensure_external_module("subunit", "subunit/python")
  32     from subunit.run import SubunitTestRunner
  33     import unittest
  34
  35 from samba.samdb import SamDB
  36 from samba.auth import system_session
  37 from ldb import Message, MessageElement, Dn, LdbError
  38 from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
  39 from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
  40
  41 parser = optparse.OptionParser("ad_dc_performance.py [options] <host>")
  42 sambaopts = options.SambaOptions(parser)
  43 parser.add_option_group(sambaopts)
  44 parser.add_option_group(options.VersionOptions(parser))
  45
  46 if not ANCIENT_SAMBA:
  47     subunitopts = SubunitOptions(parser)
  48     parser.add_option_group(subunitopts)
  49
  50 # use command line creds if available
  51 credopts = options.CredentialsOptions(parser)
  52 parser.add_option_group(credopts)
  53 opts, args = parser.parse_args()
  54
  55
  56 if len(args) < 1:
  57     parser.print_usage()
  58     sys.exit(1)
  59
  60 host = args[0]
  61
  62 lp = sambaopts.get_loadparm()
  63 creds = credopts.get_credentials(lp)
  64
  65 random.seed(1)
  66
  67
  68 class PerfTestException(Exception):
  69     pass
  70
  71
  72 BATCH_SIZE = 1000
  73 N_GROUPS = 5
  74
  75
  76 class GlobalState(object):
  77     next_user_id = 0
  78     n_groups = 0
  79     next_linked_user = 0
  80     next_relinked_user = 0
  81     next_linked_user_3 = 0
  82     next_removed_link_0 = 0
  83
  84
  85 class UserTests(samba.tests.TestCase):
  86
  87     def add_if_possible(self, *args, **kwargs):
  88         """In these tests sometimes things are left in the database
  89         deliberately, so we don't worry if we fail to add them a second
  90         time."""
  91         try:
  92             self.ldb.add(*args, **kwargs)
  93         except LdbError:
  94             pass
  95
  96     def setUp(self):
  97         super(UserTests, self).setUp()
  98         self.state = GlobalState  # the class itself, not an instance
  99         self.lp = lp
 100         self.ldb = SamDB(host, credentials=creds,
 101                          session_info=system_session(lp), lp=lp)
 102         self.base_dn = self.ldb.domain_dn()
 103         self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn)
 104         self.ou_users = "OU=users,%s" % self.ou
 105         self.ou_groups = "OU=groups,%s" % self.ou
 106         self.ou_computers = "OU=computers,%s" % self.ou
 107
 108         for dn in (self.ou, self.ou_users, self.ou_groups,
 109                    self.ou_computers):
 110             self.add_if_possible({
 111                 "dn": dn,
 112                 "objectclass": "organizationalUnit"})
 113
 114     def tearDown(self):
 115         super(UserTests, self).tearDown()
 116
 117     def test_00_00_do_nothing(self):
 118         # this gives us an idea of the overhead
 119         pass
 120
 121     def _prepare_n_groups(self, n):
 122         self.state.n_groups = n
 123         for i in range(n):
 124             self.add_if_possible({
 125                 "dn": "cn=g%d,%s" % (i, self.ou_groups),
 126                 "objectclass": "group"})
 127
 128     def _add_users(self, start, end):
 129         for i in range(start, end):
 130             self.ldb.add({
 131                 "dn": "cn=u%d,%s" % (i, self.ou_users),
 132                 "objectclass": "user"})
 133
 134     def _test_join(self):
 135         tmpdir = tempfile.mkdtemp()
 136         if '://' in host:
 137             server = host.split('://', 1)[1]
 138         else:
 139             server = host
 140         cmd = cmd_sambatool.subcommands['domain'].subcommands['join']
 141         result = cmd._run("samba-tool domain join",
 142                           creds.get_realm(),
 143                           "dc", "-U%s%%%s" % (creds.get_username(),
 144                                               creds.get_password()),
 145                           '--targetdir=%s' % tmpdir,
 146                           '--server=%s' % server)
 147
 148         shutil.rmtree(tmpdir)
 149
 150     def _test_unindexed_search(self):
 151         expressions = [
 152             ('(&(objectclass=user)(description='
 153              'Built-in account for adminstering the computer/domain))'),
 154             '(description=Built-in account for adminstering the computer/domain)',
 155             '(objectCategory=*)',
 156             '(samaccountname=Administrator*)'
 157         ]
 158         for expression in expressions:
 159             t = time.time()
 160             for i in range(10):
 161                 self.ldb.search(self.ou,
 162                                 expression=expression,
 163                                 scope=SCOPE_SUBTREE,
 164                                 attrs=['cn'])
 165             print('%d %s took %s' % (i, expression,
 166                                      time.time() - t), file=sys.stderr)
 167
 168     def _test_indexed_search(self):
 169         expressions = ['(objectclass=group)',
 170                        '(samaccountname=Administrator)'
 171                        ]
 172         for expression in expressions:
 173             t = time.time()
 174             for i in range(100):
 175                 self.ldb.search(self.ou,
 176                                 expression=expression,
 177                                 scope=SCOPE_SUBTREE,
 178                                 attrs=['cn'])
 179             print('%d runs %s took %s' % (i, expression,
 180                                           time.time() - t), file=sys.stderr)
 181
 182     def _test_add_many_users(self, n=BATCH_SIZE):
 183         s = self.state.next_user_id
 184         e = s + n
 185         self._add_users(s, e)
 186         self.state.next_user_id = e
 187
 188     test_00_00_join_empty_dc = _test_join
 189
 190     test_00_01_adding_users_1000 = _test_add_many_users
 191     test_00_02_adding_users_2000 = _test_add_many_users
 192     test_00_03_adding_users_3000 = _test_add_many_users
 193
 194     test_00_10_join_unlinked_dc = _test_join
 195     test_00_11_unindexed_search_3k_users = _test_unindexed_search
 196     test_00_12_indexed_search_3k_users = _test_indexed_search
 197
 198     def _link_user_and_group(self, u, g):
 199         m = Message()
 200         m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups))
 201         m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users),
 202                                      FLAG_MOD_ADD, "member")
 203         self.ldb.modify(m)
 204
 205     def _unlink_user_and_group(self, u, g):
 206         user = "cn=u%d,%s" % (u, self.ou_users)
 207         group = "CN=g%d,%s" % (g, self.ou_groups)
 208         m = Message()
 209         m.dn = Dn(self.ldb, group)
 210         m["member"] = MessageElement(user, FLAG_MOD_DELETE, "member")
 211         self.ldb.modify(m)
 212
 213     def _test_link_many_users(self, n=BATCH_SIZE):
 214         self._prepare_n_groups(N_GROUPS)
 215         s = self.state.next_linked_user
 216         e = s + n
 217         for i in range(s, e):
 218             g = i % N_GROUPS
 219             self._link_user_and_group(i, g)
 220         self.state.next_linked_user = e
 221
 222     test_01_01_link_users_1000 = _test_link_many_users
 223     test_01_02_link_users_2000 = _test_link_many_users
 224     test_01_03_link_users_3000 = _test_link_many_users
 225
 226     def _test_link_many_users_offset_1(self, n=BATCH_SIZE):
 227         s = self.state.next_relinked_user
 228         e = s + n
 229         for i in range(s, e):
 230             g = (i + 1) % N_GROUPS
 231             self._link_user_and_group(i, g)
 232         self.state.next_relinked_user = e
 233
 234     test_02_01_link_users_again_1000 = _test_link_many_users_offset_1
 235     test_02_02_link_users_again_2000 = _test_link_many_users_offset_1
 236     test_02_03_link_users_again_3000 = _test_link_many_users_offset_1
 237
 238     test_02_10_join_partially_linked_dc = _test_join
 239     test_02_11_unindexed_search_partially_linked_dc = _test_unindexed_search
 240     test_02_12_indexed_search_partially_linked_dc = _test_indexed_search
 241
 242     def _test_link_many_users_3_groups(self, n=BATCH_SIZE, groups=3):
 243         s = self.state.next_linked_user_3
 244         e = s + n
 245         self.state.next_linked_user_3 = e
 246         for i in range(s, e):
 247             g = (i + 2) % groups
 248             if g not in (i % N_GROUPS, (i + 1) % N_GROUPS):
 249                 self._link_user_and_group(i, g)
 250
 251     test_03_01_link_users_again_1000_few_groups = _test_link_many_users_3_groups
 252     test_03_02_link_users_again_2000_few_groups = _test_link_many_users_3_groups
 253     test_03_03_link_users_again_3000_few_groups = _test_link_many_users_3_groups
 254
 255     def _test_remove_links_0(self, n=BATCH_SIZE):
 256         s = self.state.next_removed_link_0
 257         e = s + n
 258         self.state.next_removed_link_0 = e
 259         for i in range(s, e):
 260             g = i % N_GROUPS
 261             self._unlink_user_and_group(i, g)
 262
 263     test_04_01_remove_some_links_1000 = _test_remove_links_0
 264     test_04_02_remove_some_links_2000 = _test_remove_links_0
 265     test_04_03_remove_some_links_3000 = _test_remove_links_0
 266
 267     # back to using _test_add_many_users
 268     test_05_01_adding_users_after_links_4000 = _test_add_many_users
 269
 270     # reset the link count, to replace the original links
 271     def test_06_01_relink_users_1000(self):
 272         self.state.next_linked_user = 0
 273         self._test_link_many_users()
 274
 275     test_06_02_link_users_2000 = _test_link_many_users
 276     test_06_03_link_users_3000 = _test_link_many_users
 277     test_06_04_link_users_4000 = _test_link_many_users
 278     test_06_05_link_users_again_4000 = _test_link_many_users_offset_1
 279     test_06_06_link_users_again_4000_few_groups = _test_link_many_users_3_groups
 280
 281     test_07_01_adding_users_after_links_5000 = _test_add_many_users
 282
 283     def _test_link_random_users_and_groups(self, n=BATCH_SIZE, groups=100):
 284         self._prepare_n_groups(groups)
 285         for i in range(n):
 286             u = random.randrange(self.state.next_user_id)
 287             g = random.randrange(groups)
 288             try:
 289                 self._link_user_and_group(u, g)
 290             except LdbError:
 291                 pass
 292
 293     test_08_01_link_random_users_100_groups = _test_link_random_users_and_groups
 294     test_08_02_link_random_users_100_groups = _test_link_random_users_and_groups
 295
 296     test_10_01_unindexed_search_full_dc = _test_unindexed_search
 297     test_10_02_indexed_search_full_dc = _test_indexed_search
 298     test_11_02_join_full_dc = _test_join
 299
 300     def test_20_01_delete_50_groups(self):
 301         for i in range(self.state.n_groups - 50, self.state.n_groups):
 302             self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups))
 303         self.state.n_groups -= 50
 304
 305     def _test_delete_many_users(self, n=BATCH_SIZE):
 306         e = self.state.next_user_id
 307         s = max(0, e - n)
 308         self.state.next_user_id = s
 309         for i in range(s, e):
 310             self.ldb.delete("cn=u%d,%s" % (i, self.ou_users))
 311
 312     test_21_01_delete_users_5000_lightly_linked = _test_delete_many_users
 313     test_21_02_delete_users_4000_lightly_linked = _test_delete_many_users
 314     test_21_03_delete_users_3000 = _test_delete_many_users
 315
 316     def test_22_01_delete_all_groups(self):
 317         for i in range(self.state.n_groups):
 318             self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups))
 319         self.state.n_groups = 0
 320
 321     test_23_01_delete_users_after_groups_2000 = _test_delete_many_users
 322     test_23_00_delete_users_after_groups_1000 = _test_delete_many_users
 323
 324     test_24_02_join_after_cleanup = _test_join
 325
 326
 327 if "://" not in host:
 328     if os.path.isfile(host):
 329         host = "tdb://%s" % host
 330     else:
 331         host = "ldap://%s" % host
 332
 333
 334 if ANCIENT_SAMBA:
 335     runner = SubunitTestRunner()
 336     if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
 337         sys.exit(1)
 338     sys.exit(0)
 339 else:
 340     TestProgram(module=__name__, opts=subunitopts)