PEP8: fix E127: continuation line over-indented for visual indent
[samba.git] / source4 / dsdb / tests / python / ad_dc_performance.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 from __future__ import print_function
4
5 import optparse
6 import sys
7 sys.path.insert(0, 'bin/python')
8
9 import os
10 import samba
11 import samba.getopt as options
12 import random
13 import tempfile
14 import shutil
15 import time
16
17 from samba.netcmd.main import cmd_sambatool
18
19 # We try to use the test infrastructure of Samba 4.3+, but if it
20 # doesn't work, we are probably in a back-ported patch and trying to
21 # run on 4.1 or something.
22 #
23 # Don't copy this horror into ordinary tests -- it is special for
24 # performance tests that want to apply to old versions.
25 try:
26     from samba.tests.subunitrun import SubunitOptions, TestProgram
27     ANCIENT_SAMBA = False
28 except ImportError:
29     ANCIENT_SAMBA = True
30     samba.ensure_external_module("testtools", "testtools")
31     samba.ensure_external_module("subunit", "subunit/python")
32     from subunit.run import SubunitTestRunner
33     import unittest
34
35 from samba.samdb import SamDB
36 from samba.auth import system_session
37 from ldb import Message, MessageElement, Dn, LdbError
38 from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
39 from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
40
41 parser = optparse.OptionParser("ad_dc_performance.py [options] <host>")
42 sambaopts = options.SambaOptions(parser)
43 parser.add_option_group(sambaopts)
44 parser.add_option_group(options.VersionOptions(parser))
45
46 if not ANCIENT_SAMBA:
47     subunitopts = SubunitOptions(parser)
48     parser.add_option_group(subunitopts)
49
50 # use command line creds if available
51 credopts = options.CredentialsOptions(parser)
52 parser.add_option_group(credopts)
53 opts, args = parser.parse_args()
54
55
56 if len(args) < 1:
57     parser.print_usage()
58     sys.exit(1)
59
60 host = args[0]
61
62 lp = sambaopts.get_loadparm()
63 creds = credopts.get_credentials(lp)
64
65 random.seed(1)
66
67
68 class PerfTestException(Exception):
69     pass
70
71
72 BATCH_SIZE = 1000
73 N_GROUPS = 5
74
75
76 class GlobalState(object):
77     next_user_id = 0
78     n_groups = 0
79     next_linked_user = 0
80     next_relinked_user = 0
81     next_linked_user_3 = 0
82     next_removed_link_0 = 0
83
84
85 class UserTests(samba.tests.TestCase):
86
87     def add_if_possible(self, *args, **kwargs):
88         """In these tests sometimes things are left in the database
89         deliberately, so we don't worry if we fail to add them a second
90         time."""
91         try:
92             self.ldb.add(*args, **kwargs)
93         except LdbError:
94             pass
95
96     def setUp(self):
97         super(UserTests, self).setUp()
98         self.state = GlobalState  # the class itself, not an instance
99         self.lp = lp
100         self.ldb = SamDB(host, credentials=creds,
101                          session_info=system_session(lp), lp=lp)
102         self.base_dn = self.ldb.domain_dn()
103         self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn)
104         self.ou_users = "OU=users,%s" % self.ou
105         self.ou_groups = "OU=groups,%s" % self.ou
106         self.ou_computers = "OU=computers,%s" % self.ou
107
108         for dn in (self.ou, self.ou_users, self.ou_groups,
109                    self.ou_computers):
110             self.add_if_possible({
111                 "dn": dn,
112                 "objectclass": "organizationalUnit"})
113
114     def tearDown(self):
115         super(UserTests, self).tearDown()
116
117     def test_00_00_do_nothing(self):
118         # this gives us an idea of the overhead
119         pass
120
121     def _prepare_n_groups(self, n):
122         self.state.n_groups = n
123         for i in range(n):
124             self.add_if_possible({
125                 "dn": "cn=g%d,%s" % (i, self.ou_groups),
126                 "objectclass": "group"})
127
128     def _add_users(self, start, end):
129         for i in range(start, end):
130             self.ldb.add({
131                 "dn": "cn=u%d,%s" % (i, self.ou_users),
132                 "objectclass": "user"})
133
134     def _test_join(self):
135         tmpdir = tempfile.mkdtemp()
136         if '://' in host:
137             server = host.split('://', 1)[1]
138         else:
139             server = host
140         cmd = cmd_sambatool.subcommands['domain'].subcommands['join']
141         result = cmd._run("samba-tool domain join",
142                           creds.get_realm(),
143                           "dc", "-U%s%%%s" % (creds.get_username(),
144                                               creds.get_password()),
145                           '--targetdir=%s' % tmpdir,
146                           '--server=%s' % server)
147
148         shutil.rmtree(tmpdir)
149
150     def _test_unindexed_search(self):
151         expressions = [
152             ('(&(objectclass=user)(description='
153              'Built-in account for adminstering the computer/domain))'),
154             '(description=Built-in account for adminstering the computer/domain)',
155             '(objectCategory=*)',
156             '(samaccountname=Administrator*)'
157         ]
158         for expression in expressions:
159             t = time.time()
160             for i in range(10):
161                 self.ldb.search(self.ou,
162                                 expression=expression,
163                                 scope=SCOPE_SUBTREE,
164                                 attrs=['cn'])
165             print('%d %s took %s' % (i, expression,
166                                      time.time() - t), file=sys.stderr)
167
168     def _test_indexed_search(self):
169         expressions = ['(objectclass=group)',
170                        '(samaccountname=Administrator)'
171                        ]
172         for expression in expressions:
173             t = time.time()
174             for i in range(100):
175                 self.ldb.search(self.ou,
176                                 expression=expression,
177                                 scope=SCOPE_SUBTREE,
178                                 attrs=['cn'])
179             print('%d runs %s took %s' % (i, expression,
180                                           time.time() - t), file=sys.stderr)
181
182     def _test_add_many_users(self, n=BATCH_SIZE):
183         s = self.state.next_user_id
184         e = s + n
185         self._add_users(s, e)
186         self.state.next_user_id = e
187
188     test_00_00_join_empty_dc = _test_join
189
190     test_00_01_adding_users_1000 = _test_add_many_users
191     test_00_02_adding_users_2000 = _test_add_many_users
192     test_00_03_adding_users_3000 = _test_add_many_users
193
194     test_00_10_join_unlinked_dc = _test_join
195     test_00_11_unindexed_search_3k_users = _test_unindexed_search
196     test_00_12_indexed_search_3k_users = _test_indexed_search
197
198     def _link_user_and_group(self, u, g):
199         m = Message()
200         m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups))
201         m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users),
202                                      FLAG_MOD_ADD, "member")
203         self.ldb.modify(m)
204
205     def _unlink_user_and_group(self, u, g):
206         user = "cn=u%d,%s" % (u, self.ou_users)
207         group = "CN=g%d,%s" % (g, self.ou_groups)
208         m = Message()
209         m.dn = Dn(self.ldb, group)
210         m["member"] = MessageElement(user, FLAG_MOD_DELETE, "member")
211         self.ldb.modify(m)
212
213     def _test_link_many_users(self, n=BATCH_SIZE):
214         self._prepare_n_groups(N_GROUPS)
215         s = self.state.next_linked_user
216         e = s + n
217         for i in range(s, e):
218             g = i % N_GROUPS
219             self._link_user_and_group(i, g)
220         self.state.next_linked_user = e
221
222     test_01_01_link_users_1000 = _test_link_many_users
223     test_01_02_link_users_2000 = _test_link_many_users
224     test_01_03_link_users_3000 = _test_link_many_users
225
226     def _test_link_many_users_offset_1(self, n=BATCH_SIZE):
227         s = self.state.next_relinked_user
228         e = s + n
229         for i in range(s, e):
230             g = (i + 1) % N_GROUPS
231             self._link_user_and_group(i, g)
232         self.state.next_relinked_user = e
233
234     test_02_01_link_users_again_1000 = _test_link_many_users_offset_1
235     test_02_02_link_users_again_2000 = _test_link_many_users_offset_1
236     test_02_03_link_users_again_3000 = _test_link_many_users_offset_1
237
238     test_02_10_join_partially_linked_dc = _test_join
239     test_02_11_unindexed_search_partially_linked_dc = _test_unindexed_search
240     test_02_12_indexed_search_partially_linked_dc = _test_indexed_search
241
242     def _test_link_many_users_3_groups(self, n=BATCH_SIZE, groups=3):
243         s = self.state.next_linked_user_3
244         e = s + n
245         self.state.next_linked_user_3 = e
246         for i in range(s, e):
247             g = (i + 2) % groups
248             if g not in (i % N_GROUPS, (i + 1) % N_GROUPS):
249                 self._link_user_and_group(i, g)
250
251     test_03_01_link_users_again_1000_few_groups = _test_link_many_users_3_groups
252     test_03_02_link_users_again_2000_few_groups = _test_link_many_users_3_groups
253     test_03_03_link_users_again_3000_few_groups = _test_link_many_users_3_groups
254
255     def _test_remove_links_0(self, n=BATCH_SIZE):
256         s = self.state.next_removed_link_0
257         e = s + n
258         self.state.next_removed_link_0 = e
259         for i in range(s, e):
260             g = i % N_GROUPS
261             self._unlink_user_and_group(i, g)
262
263     test_04_01_remove_some_links_1000 = _test_remove_links_0
264     test_04_02_remove_some_links_2000 = _test_remove_links_0
265     test_04_03_remove_some_links_3000 = _test_remove_links_0
266
267     # back to using _test_add_many_users
268     test_05_01_adding_users_after_links_4000 = _test_add_many_users
269
270     # reset the link count, to replace the original links
271     def test_06_01_relink_users_1000(self):
272         self.state.next_linked_user = 0
273         self._test_link_many_users()
274
275     test_06_02_link_users_2000 = _test_link_many_users
276     test_06_03_link_users_3000 = _test_link_many_users
277     test_06_04_link_users_4000 = _test_link_many_users
278     test_06_05_link_users_again_4000 = _test_link_many_users_offset_1
279     test_06_06_link_users_again_4000_few_groups = _test_link_many_users_3_groups
280
281     test_07_01_adding_users_after_links_5000 = _test_add_many_users
282
283     def _test_link_random_users_and_groups(self, n=BATCH_SIZE, groups=100):
284         self._prepare_n_groups(groups)
285         for i in range(n):
286             u = random.randrange(self.state.next_user_id)
287             g = random.randrange(groups)
288             try:
289                 self._link_user_and_group(u, g)
290             except LdbError:
291                 pass
292
293     test_08_01_link_random_users_100_groups = _test_link_random_users_and_groups
294     test_08_02_link_random_users_100_groups = _test_link_random_users_and_groups
295
296     test_10_01_unindexed_search_full_dc = _test_unindexed_search
297     test_10_02_indexed_search_full_dc = _test_indexed_search
298     test_11_02_join_full_dc = _test_join
299
300     def test_20_01_delete_50_groups(self):
301         for i in range(self.state.n_groups - 50, self.state.n_groups):
302             self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups))
303         self.state.n_groups -= 50
304
305     def _test_delete_many_users(self, n=BATCH_SIZE):
306         e = self.state.next_user_id
307         s = max(0, e - n)
308         self.state.next_user_id = s
309         for i in range(s, e):
310             self.ldb.delete("cn=u%d,%s" % (i, self.ou_users))
311
312     test_21_01_delete_users_5000_lightly_linked = _test_delete_many_users
313     test_21_02_delete_users_4000_lightly_linked = _test_delete_many_users
314     test_21_03_delete_users_3000 = _test_delete_many_users
315
316     def test_22_01_delete_all_groups(self):
317         for i in range(self.state.n_groups):
318             self.ldb.delete("cn=g%d,%s" % (i, self.ou_groups))
319         self.state.n_groups = 0
320
321     test_23_01_delete_users_after_groups_2000 = _test_delete_many_users
322     test_23_00_delete_users_after_groups_1000 = _test_delete_many_users
323
324     test_24_02_join_after_cleanup = _test_join
325
326
327 if "://" not in host:
328     if os.path.isfile(host):
329         host = "tdb://%s" % host
330     else:
331         host = "ldap://%s" % host
332
333
334 if ANCIENT_SAMBA:
335     runner = SubunitTestRunner()
336     if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
337         sys.exit(1)
338     sys.exit(0)
339 else:
340     TestProgram(module=__name__, opts=subunitopts)