PY3: change shebang to python3 in wintest dir
[amitay/samba.git] / wintest / wintest.py
1 #!/usr/bin/env python3
2
3 '''automated testing library for testing Samba against windows'''
4
5 import pexpect
6 import subprocess
7 import optparse
8 import sys
9 import os
10 import time
11 import re
12
13
14 class wintest():
15     '''testing of Samba against windows VMs'''
16
17     def __init__(self):
18         self.vars = {}
19         self.list_mode = False
20         self.vms = None
21         os.environ['PYTHONUNBUFFERED'] = '1'
22         self.parser = optparse.OptionParser("wintest")
23
24     def check_prerequesites(self):
25         self.info("Checking prerequesites")
26         self.setvar('HOSTNAME', self.cmd_output("hostname -s").strip())
27         if os.getuid() != 0:
28             raise Exception("You must run this script as root")
29         self.run_cmd('ifconfig ${INTERFACE} ${INTERFACE_NET} up')
30         if self.getvar('INTERFACE_IPV6'):
31             self.run_cmd('ifconfig ${INTERFACE} inet6 del ${INTERFACE_IPV6}/64', checkfail=False)
32             self.run_cmd('ifconfig ${INTERFACE} inet6 add ${INTERFACE_IPV6}/64 up')
33
34         self.run_cmd('ifconfig ${NAMED_INTERFACE} ${NAMED_INTERFACE_NET} up')
35         if self.getvar('NAMED_INTERFACE_IPV6'):
36             self.run_cmd('ifconfig ${NAMED_INTERFACE} inet6 del ${NAMED_INTERFACE_IPV6}/64', checkfail=False)
37             self.run_cmd('ifconfig ${NAMED_INTERFACE} inet6 add ${NAMED_INTERFACE_IPV6}/64 up')
38
39     def stop_vms(self):
40         '''Shut down any existing alive VMs, so they do not collide with what we are doing'''
41         self.info('Shutting down any of our VMs already running')
42         vms = self.get_vms()
43         for v in vms:
44             self.vm_poweroff(v, checkfail=False)
45
46     def setvar(self, varname, value):
47         '''set a substitution variable'''
48         self.vars[varname] = value
49
50     def getvar(self, varname):
51         '''return a substitution variable'''
52         if varname not in self.vars:
53             return None
54         return self.vars[varname]
55
56     def setwinvars(self, vm, prefix='WIN'):
57         '''setup WIN_XX vars based on a vm name'''
58         for v in ['VM', 'HOSTNAME', 'USER', 'PASS', 'SNAPSHOT', 'REALM', 'DOMAIN', 'IP']:
59             vname = '%s_%s' % (vm, v)
60             if vname in self.vars:
61                 self.setvar("%s_%s" % (prefix, v), self.substitute("${%s}" % vname))
62             else:
63                 self.vars.pop("%s_%s" % (prefix, v), None)
64
65         if self.getvar("WIN_REALM"):
66             self.setvar("WIN_REALM", self.getvar("WIN_REALM").upper())
67             self.setvar("WIN_LCREALM", self.getvar("WIN_REALM").lower())
68             dnsdomain = self.getvar("WIN_REALM")
69             self.setvar("WIN_BASEDN", "DC=" + dnsdomain.replace(".", ",DC="))
70         if self.getvar("WIN_USER") is None:
71             self.setvar("WIN_USER", "administrator")
72
73     def info(self, msg):
74         '''print some information'''
75         if not self.list_mode:
76             print(self.substitute(msg))
77
78     def load_config(self, fname):
79         '''load the config file'''
80         f = open(fname)
81         for line in f:
82             line = line.strip()
83             if len(line) == 0 or line[0] == '#':
84                 continue
85             colon = line.find(':')
86             if colon == -1:
87                 raise RuntimeError("Invalid config line '%s'" % line)
88             varname = line[0:colon].strip()
89             value   = line[colon + 1:].strip()
90             self.setvar(varname, value)
91
92     def list_steps_mode(self):
93         '''put wintest in step listing mode'''
94         self.list_mode = True
95
96     def set_skip(self, skiplist):
97         '''set a list of tests to skip'''
98         self.skiplist = skiplist.split(',')
99
100     def set_vms(self, vms):
101         '''set a list of VMs to test'''
102         if vms is not None:
103             self.vms = []
104             for vm in vms.split(','):
105                 vm = vm.upper()
106                 self.vms.append(vm)
107
108     def skip(self, step):
109         '''return True if we should skip a step'''
110         if self.list_mode:
111             print("\t%s" % step)
112             return True
113         return step in self.skiplist
114
115     def substitute(self, text):
116         """Substitute strings of the form ${NAME} in text, replacing
117         with substitutions from vars.
118         """
119         if isinstance(text, list):
120             ret = text[:]
121             for i in range(len(ret)):
122                 ret[i] = self.substitute(ret[i])
123             return ret
124
125         """We may have objects such as pexpect.EOF that are not strings"""
126         if not isinstance(text, str):
127             return text
128         while True:
129             var_start = text.find("${")
130             if var_start == -1:
131                 return text
132             var_end = text.find("}", var_start)
133             if var_end == -1:
134                 return text
135             var_name = text[var_start + 2:var_end]
136             if var_name not in self.vars:
137                 raise RuntimeError("Unknown substitution variable ${%s}" % var_name)
138             text = text.replace("${%s}" % var_name, self.vars[var_name])
139         return text
140
141     def have_var(self, varname):
142         '''see if a variable has been set'''
143         return varname in self.vars
144
145     def have_vm(self, vmname):
146         '''see if a VM should be used'''
147         if not self.have_var(vmname + '_VM'):
148             return False
149         if self.vms is None:
150             return True
151         return vmname in self.vms
152
153     def putenv(self, key, value):
154         '''putenv with substitution'''
155         os.environ[key] = self.substitute(value)
156
157     def chdir(self, dir):
158         '''chdir with substitution'''
159         os.chdir(self.substitute(dir))
160
161     def del_files(self, dirs):
162         '''delete all files in the given directory'''
163         for d in dirs:
164             self.run_cmd("find %s -type f | xargs rm -f" % d)
165
166     def write_file(self, filename, text, mode='w'):
167         '''write to a file'''
168         f = open(self.substitute(filename), mode=mode)
169         f.write(self.substitute(text))
170         f.close()
171
172     def run_cmd(self, cmd, dir=".", show=None, output=False, checkfail=True):
173         '''run a command'''
174         cmd = self.substitute(cmd)
175         if isinstance(cmd, list):
176             self.info('$ ' + " ".join(cmd))
177         else:
178             self.info('$ ' + cmd)
179         if output:
180             return subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=dir).communicate()[0]
181         if isinstance(cmd, list):
182             shell = False
183         else:
184             shell = True
185         if checkfail:
186             return subprocess.check_call(cmd, shell=shell, cwd=dir)
187         else:
188             return subprocess.call(cmd, shell=shell, cwd=dir)
189
190     def run_child(self, cmd, dir="."):
191         '''create a child and return the Popen handle to it'''
192         cwd = os.getcwd()
193         cmd = self.substitute(cmd)
194         if isinstance(cmd, list):
195             self.info('$ ' + " ".join(cmd))
196         else:
197             self.info('$ ' + cmd)
198         if isinstance(cmd, list):
199             shell = False
200         else:
201             shell = True
202         os.chdir(dir)
203         ret = subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT)
204         os.chdir(cwd)
205         return ret
206
207     def cmd_output(self, cmd):
208         '''return output from and command'''
209         cmd = self.substitute(cmd)
210         return self.run_cmd(cmd, output=True)
211
212     def cmd_contains(self, cmd, contains, nomatch=False, ordered=False, regex=False,
213                      casefold=True):
214         '''check that command output contains the listed strings'''
215
216         if isinstance(contains, str):
217             contains = [contains]
218
219         out = self.cmd_output(cmd)
220         self.info(out)
221         for c in self.substitute(contains):
222             if regex:
223                 if casefold:
224                     c = c.upper()
225                     out = out.upper()
226                 m = re.search(c, out)
227                 if m is None:
228                     start = -1
229                     end = -1
230                 else:
231                     start = m.start()
232                     end = m.end()
233             elif casefold:
234                 start = out.upper().find(c.upper())
235                 end = start + len(c)
236             else:
237                 start = out.find(c)
238                 end = start + len(c)
239             if nomatch:
240                 if start != -1:
241                     raise RuntimeError("Expected to not see %s in %s" % (c, cmd))
242             else:
243                 if start == -1:
244                     raise RuntimeError("Expected to see %s in %s" % (c, cmd))
245             if ordered and start != -1:
246                 out = out[end:]
247
248     def retry_cmd(self, cmd, contains, retries=30, delay=2, wait_for_fail=False,
249                   ordered=False, regex=False, casefold=True):
250         '''retry a command a number of times'''
251         while retries > 0:
252             try:
253                 self.cmd_contains(cmd, contains, nomatch=wait_for_fail,
254                                   ordered=ordered, regex=regex, casefold=casefold)
255                 return
256             except:
257                 time.sleep(delay)
258                 retries -= 1
259                 self.info("retrying (retries=%u delay=%u)" % (retries, delay))
260         raise RuntimeError("Failed to find %s" % contains)
261
262     def pexpect_spawn(self, cmd, timeout=60, crlf=True, casefold=True):
263         '''wrapper around pexpect spawn'''
264         cmd = self.substitute(cmd)
265         self.info("$ " + cmd)
266         ret = pexpect.spawn(cmd, logfile=sys.stdout, timeout=timeout)
267
268         def sendline_sub(line):
269             line = self.substitute(line)
270             if crlf:
271                 line = line.replace('\n', '\r\n') + '\r'
272             return ret.old_sendline(line)
273
274         def expect_sub(line, timeout=ret.timeout, casefold=casefold):
275             line = self.substitute(line)
276             if casefold:
277                 if isinstance(line, list):
278                     for i in range(len(line)):
279                         if isinstance(line[i], str):
280                             line[i] = '(?i)' + line[i]
281                 elif isinstance(line, str):
282                     line = '(?i)' + line
283             return ret.old_expect(line, timeout=timeout)
284
285         ret.old_sendline = ret.sendline
286         ret.sendline = sendline_sub
287         ret.old_expect = ret.expect
288         ret.expect = expect_sub
289
290         return ret
291
292     def get_nameserver(self):
293         '''Get the current nameserver from /etc/resolv.conf'''
294         child = self.pexpect_spawn('cat /etc/resolv.conf', crlf=False)
295         i = child.expect(['Generated by wintest', 'nameserver'])
296         if i == 0:
297             child.expect('your original resolv.conf')
298             child.expect('nameserver')
299         child.expect('\d+.\d+.\d+.\d+')
300         return child.after
301
302     def rndc_cmd(self, cmd, checkfail=True):
303         '''run a rndc command'''
304         self.run_cmd("${RNDC} -c ${PREFIX}/etc/rndc.conf %s" % cmd, checkfail=checkfail)
305
306     def named_supports_gssapi_keytab(self):
307         '''see if named supports tkey-gssapi-keytab'''
308         self.write_file("${PREFIX}/named.conf.test",
309                         'options { tkey-gssapi-keytab "test"; };')
310         try:
311             self.run_cmd("${NAMED_CHECKCONF} ${PREFIX}/named.conf.test")
312         except subprocess.CalledProcessError:
313             return False
314         return True
315
316     def set_nameserver(self, nameserver):
317         '''set the nameserver in resolv.conf'''
318         self.write_file("/etc/resolv.conf.wintest", '''
319 # Generated by wintest, the Samba v Windows automated testing system
320 nameserver %s
321
322 # your original resolv.conf appears below:
323 ''' % self.substitute(nameserver))
324         child = self.pexpect_spawn("cat /etc/resolv.conf", crlf=False)
325         i = child.expect(['your original resolv.conf appears below:', pexpect.EOF])
326         if i == 0:
327             child.expect(pexpect.EOF)
328         contents = child.before.lstrip().replace('\r', '')
329         self.write_file('/etc/resolv.conf.wintest', contents, mode='a')
330         self.write_file('/etc/resolv.conf.wintest-bak', contents)
331         self.run_cmd("mv -f /etc/resolv.conf.wintest /etc/resolv.conf")
332         self.resolv_conf_backup = '/etc/resolv.conf.wintest-bak'
333
334     def configure_bind(self, kerberos_support=False, include=None):
335         self.chdir('${PREFIX}')
336
337         if self.getvar('NAMED_INTERFACE_IPV6'):
338             ipv6_listen = 'listen-on-v6 port 53 { ${NAMED_INTERFACE_IPV6}; };'
339         else:
340             ipv6_listen = ''
341         self.setvar('BIND_LISTEN_IPV6', ipv6_listen)
342
343         if not kerberos_support:
344             self.setvar("NAMED_TKEY_OPTION", "")
345         elif self.getvar('NAMESERVER_BACKEND') != 'SAMBA_INTERNAL':
346             if self.named_supports_gssapi_keytab():
347                 self.setvar("NAMED_TKEY_OPTION",
348                             'tkey-gssapi-keytab "${PREFIX}/bind-dns/dns.keytab";')
349             else:
350                 self.info("LCREALM=${LCREALM}")
351                 self.setvar("NAMED_TKEY_OPTION",
352                             '''tkey-gssapi-credential "DNS/${LCREALM}";
353                             tkey-domain "${LCREALM}";
354                  ''')
355             self.putenv('KEYTAB_FILE', '${PREFIX}/bind-dns/dns.keytab')
356             self.putenv('KRB5_KTNAME', '${PREFIX}/bind-dns/dns.keytab')
357         else:
358             self.setvar("NAMED_TKEY_OPTION", "")
359
360         if include and self.getvar('NAMESERVER_BACKEND') != 'SAMBA_INTERNAL':
361             self.setvar("NAMED_INCLUDE", 'include "%s";' % include)
362         else:
363             self.setvar("NAMED_INCLUDE", '')
364
365         self.run_cmd("mkdir -p ${PREFIX}/etc")
366
367         self.write_file("etc/named.conf", '''
368 options {
369         listen-on port 53 { ${NAMED_INTERFACE_IP};  };
370         ${BIND_LISTEN_IPV6}
371         directory       "${PREFIX}/var/named";
372         dump-file       "${PREFIX}/var/named/data/cache_dump.db";
373         pid-file        "${PREFIX}/var/named/named.pid";
374         statistics-file "${PREFIX}/var/named/data/named_stats.txt";
375         memstatistics-file "${PREFIX}/var/named/data/named_mem_stats.txt";
376         allow-query     { any; };
377         recursion yes;
378         ${NAMED_TKEY_OPTION}
379         max-cache-ttl 10;
380         max-ncache-ttl 10;
381
382         forward only;
383         forwarders {
384                   ${DNSSERVER};
385         };
386
387 };
388
389 key "rndc-key" {
390         algorithm hmac-md5;
391         secret "lA/cTrno03mt5Ju17ybEYw==";
392 };
393
394 controls {
395         inet ${NAMED_INTERFACE_IP} port 953
396         allow { any; } keys { "rndc-key"; };
397 };
398
399 ${NAMED_INCLUDE}
400 ''')
401
402         if self.getvar('NAMESERVER_BACKEND') == 'SAMBA_INTERNAL':
403             self.write_file('etc/named.conf',
404                             '''
405 zone "%s" IN {
406       type forward;
407       forward only;
408       forwarders {
409          %s;
410       };
411 };
412 ''' % (self.getvar('LCREALM'), self.getvar('INTERFACE_IP')),
413                    mode='a')
414
415         # add forwarding for the windows domains
416         domains = self.get_domains()
417
418         for d in domains:
419             self.write_file('etc/named.conf',
420                             '''
421 zone "%s" IN {
422       type forward;
423       forward only;
424       forwarders {
425          %s;
426       };
427 };
428 ''' % (d, domains[d]),
429                      mode='a')
430
431         self.write_file("etc/rndc.conf", '''
432 # Start of rndc.conf
433 key "rndc-key" {
434         algorithm hmac-md5;
435         secret "lA/cTrno03mt5Ju17ybEYw==";
436 };
437
438 options {
439         default-key "rndc-key";
440         default-server  ${NAMED_INTERFACE_IP};
441         default-port 953;
442 };
443 ''')
444
445     def stop_bind(self):
446         '''Stop our private BIND from listening and operating'''
447         self.rndc_cmd("stop", checkfail=False)
448         self.port_wait("${NAMED_INTERFACE_IP}", 53, wait_for_fail=True)
449
450         self.run_cmd("rm -rf var/named")
451
452     def start_bind(self):
453         '''restart the test environment version of bind'''
454         self.info("Restarting bind9")
455         self.chdir('${PREFIX}')
456
457         self.set_nameserver(self.getvar('NAMED_INTERFACE_IP'))
458
459         self.run_cmd("mkdir -p var/named/data")
460         self.run_cmd("chown -R ${BIND_USER} var/named")
461
462         self.bind_child = self.run_child("${BIND9} -u ${BIND_USER} -n 1 -c ${PREFIX}/etc/named.conf -g")
463
464         self.port_wait("${NAMED_INTERFACE_IP}", 53)
465         self.rndc_cmd("flush")
466
467     def restart_bind(self, kerberos_support=False, include=None):
468         self.configure_bind(kerberos_support=kerberos_support, include=include)
469         self.stop_bind()
470         self.start_bind()
471
472     def restore_resolv_conf(self):
473         '''restore the /etc/resolv.conf after testing is complete'''
474         if getattr(self, 'resolv_conf_backup', False):
475             self.info("restoring /etc/resolv.conf")
476             self.run_cmd("mv -f %s /etc/resolv.conf" % self.resolv_conf_backup)
477
478     def vm_poweroff(self, vmname, checkfail=True):
479         '''power off a VM'''
480         self.setvar('VMNAME', vmname)
481         self.run_cmd("${VM_POWEROFF}", checkfail=checkfail)
482
483     def vm_reset(self, vmname):
484         '''reset a VM'''
485         self.setvar('VMNAME', vmname)
486         self.run_cmd("${VM_RESET}")
487
488     def vm_restore(self, vmname, snapshot):
489         '''restore a VM'''
490         self.setvar('VMNAME', vmname)
491         self.setvar('SNAPSHOT', snapshot)
492         self.run_cmd("${VM_RESTORE}")
493
494     def ping_wait(self, hostname):
495         '''wait for a hostname to come up on the network'''
496         hostname = self.substitute(hostname)
497         loops = 10
498         while loops > 0:
499             try:
500                 self.run_cmd("ping -c 1 -w 10 %s" % hostname)
501                 break
502             except:
503                 loops = loops - 1
504         if loops == 0:
505             raise RuntimeError("Failed to ping %s" % hostname)
506         self.info("Host %s is up" % hostname)
507
508     def port_wait(self, hostname, port, retries=200, delay=3, wait_for_fail=False):
509         '''wait for a host to come up on the network'''
510
511         while retries > 0:
512             child = self.pexpect_spawn("nc -v -z -w 1 %s %u" % (hostname, port), crlf=False, timeout=1)
513             child.expect([pexpect.EOF, pexpect.TIMEOUT])
514             child.close()
515             i = child.exitstatus
516             if wait_for_fail:
517                 # wait for timeout or fail
518                 if i is None or i > 0:
519                     return
520             else:
521                 if i == 0:
522                     return
523
524             time.sleep(delay)
525             retries -= 1
526             self.info("retrying (retries=%u delay=%u)" % (retries, delay))
527
528         raise RuntimeError("gave up waiting for %s:%d" % (hostname, port))
529
530     def run_net_time(self, child):
531         '''run net time on windows'''
532         child.sendline("net time \\\\${HOSTNAME} /set")
533         child.expect("Do you want to set the local computer")
534         child.sendline("Y")
535         child.expect("The command completed successfully")
536
537     def run_date_time(self, child, time_tuple=None):
538         '''run date and time on windows'''
539         if time_tuple is None:
540             time_tuple = time.localtime()
541         child.sendline("date")
542         child.expect("Enter the new date:")
543         i = child.expect(["dd-mm-yy", "mm-dd-yy"])
544         if i == 0:
545             child.sendline(time.strftime("%d-%m-%y", time_tuple))
546         else:
547             child.sendline(time.strftime("%m-%d-%y", time_tuple))
548         child.expect("C:")
549         child.sendline("time")
550         child.expect("Enter the new time:")
551         child.sendline(time.strftime("%H:%M:%S", time_tuple))
552         child.expect("C:")
553
554     def get_ipconfig(self, child):
555         '''get the IP configuration of the child'''
556         child.sendline("ipconfig /all")
557         child.expect('Ethernet adapter ')
558         child.expect("[\w\s]+")
559         self.setvar("WIN_NIC", child.after)
560         child.expect(['IPv4 Address', 'IP Address'])
561         child.expect('\d+.\d+.\d+.\d+')
562         self.setvar('WIN_IPV4_ADDRESS', child.after)
563         child.expect('Subnet Mask')
564         child.expect('\d+.\d+.\d+.\d+')
565         self.setvar('WIN_SUBNET_MASK', child.after)
566         child.expect('Default Gateway')
567         i = child.expect(['\d+.\d+.\d+.\d+', "C:"])
568         if i == 0:
569             self.setvar('WIN_DEFAULT_GATEWAY', child.after)
570             child.expect("C:")
571
572     def get_is_dc(self, child):
573         '''check if a windows machine is a domain controller'''
574         child.sendline("dcdiag")
575         i = child.expect(["is not a [Directory Server|DC]",
576                           "is not recognized as an internal or external command",
577                           "Home Server = ",
578                           "passed test Replications"])
579         if i == 0:
580             return False
581         if i == 1 or i == 3:
582             child.expect("C:")
583             child.sendline("net config Workstation")
584             child.expect("Workstation domain")
585             child.expect('[\S]+')
586             domain = child.after
587             i = child.expect(["Workstation Domain DNS Name", "Logon domain"])
588             '''If we get the Logon domain first, we are not in an AD domain'''
589             if i == 1:
590                 return False
591             if domain.upper() == self.getvar("WIN_DOMAIN").upper():
592                 return True
593
594         child.expect('[\S]+')
595         hostname = child.after
596         if hostname.upper() == self.getvar("WIN_HOSTNAME").upper():
597             return True
598
599     def set_noexpire(self, child, username):
600         """Ensure this user's password does not expire"""
601         child.sendline('wmic useraccount where name="%s" set PasswordExpires=FALSE' % username)
602         child.expect("update successful")
603         child.expect("C:")
604
605     def run_tlntadmn(self, child):
606         '''remove the annoying telnet restrictions'''
607         child.sendline('tlntadmn config maxconn=1024')
608         child.expect(["The settings were successfully updated", "Access is denied"])
609         child.expect("C:")
610
611     def disable_firewall(self, child):
612         '''remove the annoying firewall'''
613         child.sendline('netsh advfirewall set allprofiles state off')
614         i = child.expect(["Ok", "The following command was not found: advfirewall set allprofiles state off", "The requested operation requires elevation", "Access is denied"])
615         child.expect("C:")
616         if i == 1:
617             child.sendline('netsh firewall set opmode mode = DISABLE profile = ALL')
618             i = child.expect(["Ok", "The following command was not found", "Access is denied"])
619             if i != 0:
620                 self.info("Firewall disable failed - ignoring")
621             child.expect("C:")
622
623     def set_dns(self, child):
624         child.sendline('netsh interface ip set dns "${WIN_NIC}" static ${NAMED_INTERFACE_IP} primary')
625         i = child.expect(['C:', pexpect.EOF, pexpect.TIMEOUT], timeout=5)
626         if i > 0:
627             return True
628         else:
629             return False
630
631     def set_ip(self, child):
632         """fix the IP address to the same value it had when we
633         connected, but don't use DHCP, and force the DNS server to our
634         DNS server.  This allows DNS updates to run"""
635         self.get_ipconfig(child)
636         if self.getvar("WIN_IPV4_ADDRESS") != self.getvar("WIN_IP"):
637             raise RuntimeError("ipconfig address %s != nmblookup address %s" % (self.getvar("WIN_IPV4_ADDRESS"),
638                                                                                 self.getvar("WIN_IP")))
639         child.sendline('netsh')
640         child.expect('netsh>')
641         child.sendline('offline')
642         child.expect('netsh>')
643         child.sendline('routing ip add persistentroute dest=0.0.0.0 mask=0.0.0.0 name="${WIN_NIC}" nhop=${WIN_DEFAULT_GATEWAY}')
644         child.expect('netsh>')
645         child.sendline('interface ip set address "${WIN_NIC}" static ${WIN_IPV4_ADDRESS} ${WIN_SUBNET_MASK} ${WIN_DEFAULT_GATEWAY} 1 store=persistent')
646         i = child.expect(['The syntax supplied for this command is not valid. Check help for the correct syntax', 'netsh>', pexpect.EOF, pexpect.TIMEOUT], timeout=5)
647         if i == 0:
648             child.sendline('interface ip set address "${WIN_NIC}" static ${WIN_IPV4_ADDRESS} ${WIN_SUBNET_MASK} ${WIN_DEFAULT_GATEWAY} 1')
649             child.expect('netsh>')
650         child.sendline('commit')
651         child.sendline('online')
652         child.sendline('exit')
653
654         child.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=5)
655         return True
656
657     def resolve_ip(self, hostname, retries=60, delay=5):
658         '''resolve an IP given a hostname, assuming NBT'''
659         while retries > 0:
660             child = self.pexpect_spawn("bin/nmblookup %s" % hostname)
661             i = 0
662             while i == 0:
663                 i = child.expect(["querying", '\d+.\d+.\d+.\d+', hostname, "Lookup failed"])
664                 if i == 0:
665                     child.expect("\r")
666             if i == 1:
667                 return child.after
668             retries -= 1
669             time.sleep(delay)
670             self.info("retrying (retries=%u delay=%u)" % (retries, delay))
671         raise RuntimeError("Failed to resolve IP of %s" % hostname)
672
673     def open_telnet(self, hostname, username, password, retries=60, delay=5, set_time=False, set_ip=False,
674                     disable_firewall=True, run_tlntadmn=True, set_noexpire=False):
675         '''open a telnet connection to a windows server, return the pexpect child'''
676         set_route = False
677         set_dns = False
678         set_telnetclients = True
679         start_telnet = True
680         if self.getvar('WIN_IP'):
681             ip = self.getvar('WIN_IP')
682         else:
683             ip = self.resolve_ip(hostname)
684             self.setvar('WIN_IP', ip)
685         while retries > 0:
686             child = self.pexpect_spawn("telnet " + ip + " -l '" + username + "'")
687             i = child.expect(["Welcome to Microsoft Telnet Service",
688                               "Denying new connections due to the limit on number of connections",
689                               "No more connections are allowed to telnet server",
690                               "Unable to connect to remote host",
691                               "No route to host",
692                               "Connection refused",
693                               pexpect.EOF])
694             if i != 0:
695                 child.close()
696                 time.sleep(delay)
697                 retries -= 1
698                 self.info("retrying (retries=%u delay=%u)" % (retries, delay))
699                 continue
700             child.expect("password:")
701             child.sendline(password)
702             i = child.expect(["C:",
703                               "TelnetClients",
704                               "Denying new connections due to the limit on number of connections",
705                               "No more connections are allowed to telnet server",
706                               "Unable to connect to remote host",
707                               "No route to host",
708                               "Connection refused",
709                               pexpect.EOF])
710             if i == 1:
711                 if set_telnetclients:
712                     self.run_cmd('bin/net rpc group add TelnetClients -S $WIN_IP -U$WIN_USER%$WIN_PASS')
713                     self.run_cmd('bin/net rpc group addmem TelnetClients "authenticated users" -S $WIN_IP -U$WIN_USER%$WIN_PASS')
714                     child.close()
715                     retries -= 1
716                     set_telnetclients = False
717                     self.info("retrying (retries=%u delay=%u)" % (retries, delay))
718                     continue
719                 else:
720                     raise RuntimeError("Failed to connect with telnet due to missing TelnetClients membership")
721
722             if i == 6:
723                 # This only works if it is installed and enabled, but not started.  Not entirely likely, but possible
724                 self.run_cmd('bin/net rpc service start TlntSvr -S $WIN_IP -U$WIN_USER%$WIN_PASS')
725                 child.close()
726                 start_telnet = False
727                 retries -= 1
728                 self.info("retrying (retries=%u delay=%u)" % (retries, delay))
729                 continue
730
731             if i != 0:
732                 child.close()
733                 time.sleep(delay)
734                 retries -= 1
735                 self.info("retrying (retries=%u delay=%u)" % (retries, delay))
736                 continue
737             if set_dns:
738                 set_dns = False
739                 if self.set_dns(child):
740                     continue
741             if set_route:
742                 child.sendline('route add 0.0.0.0 mask 0.0.0.0 ${WIN_DEFAULT_GATEWAY}')
743                 child.expect("C:")
744                 set_route = False
745             if set_time:
746                 self.run_date_time(child, None)
747                 set_time = False
748             if run_tlntadmn:
749                 self.run_tlntadmn(child)
750                 run_tlntadmn = False
751             if set_noexpire:
752                 self.set_noexpire(child, username)
753                 set_noexpire = False
754             if disable_firewall:
755                 self.disable_firewall(child)
756                 disable_firewall = False
757             if set_ip:
758                 set_ip = False
759                 if self.set_ip(child):
760                     set_route = True
761                     set_dns = True
762                 continue
763             return child
764         raise RuntimeError("Failed to connect with telnet")
765
766     def kinit(self, username, password):
767         '''use kinit to setup a credentials cache'''
768         self.run_cmd("kdestroy")
769         self.putenv('KRB5CCNAME', "${PREFIX}/ccache.test")
770         username = self.substitute(username)
771         s = username.split('@')
772         if len(s) > 0:
773             s[1] = s[1].upper()
774         username = '@'.join(s)
775         child = self.pexpect_spawn('kinit ' + username)
776         child.expect("Password")
777         child.sendline(password)
778         child.expect(pexpect.EOF)
779         child.close()
780         if child.exitstatus != 0:
781             raise RuntimeError("kinit failed with status %d" % child.exitstatus)
782
783     def get_domains(self):
784         '''return a dictionary of DNS domains and IPs for named.conf'''
785         ret = {}
786         for v in self.vars:
787             if v[-6:] == "_REALM":
788                 base = v[:-6]
789                 if base + '_IP' in self.vars:
790                     ret[self.vars[base + '_REALM']] = self.vars[base + '_IP']
791         return ret
792
793     def wait_reboot(self, retries=3):
794         '''wait for a VM to reboot'''
795
796         # first wait for it to shutdown
797         self.port_wait("${WIN_IP}", 139, wait_for_fail=True, delay=6)
798
799         # now wait for it to come back. If it fails to come back
800         # then try resetting it
801         while retries > 0:
802             try:
803                 self.port_wait("${WIN_IP}", 139)
804                 return
805             except:
806                 retries -= 1
807                 self.vm_reset("${WIN_VM}")
808                 self.info("retrying reboot (retries=%u)" % retries)
809         raise RuntimeError(self.substitute("VM ${WIN_VM} failed to reboot"))
810
811     def get_vms(self):
812         '''return a dictionary of all the configured VM names'''
813         ret = []
814         for v in self.vars:
815             if v[-3:] == "_VM":
816                 ret.append(self.vars[v])
817         return ret
818
819     def run_dcpromo_as_first_dc(self, vm, func_level=None):
820         self.setwinvars(vm)
821         self.info("Configuring a windows VM ${WIN_VM} at the first DC in the domain using dcpromo")
822         child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_time=True)
823         if self.get_is_dc(child):
824             return
825
826         if func_level == '2008r2':
827             self.setvar("FUNCTION_LEVEL_INT", str(4))
828         elif func_level == '2003':
829             self.setvar("FUNCTION_LEVEL_INT", str(1))
830         else:
831             self.setvar("FUNCTION_LEVEL_INT", str(0))
832
833         child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_ip=True, set_noexpire=True)
834
835         """This server must therefore not yet be a directory server, so we must promote it"""
836         child.sendline("copy /Y con answers.txt")
837         child.sendline('''
838 [DCInstall]
839 ; New forest promotion
840 ReplicaOrNewDomain=Domain
841 NewDomain=Forest
842 NewDomainDNSName=${WIN_REALM}
843 ForestLevel=${FUNCTION_LEVEL_INT}
844 DomainNetbiosName=${WIN_DOMAIN}
845 DomainLevel=${FUNCTION_LEVEL_INT}
846 InstallDNS=Yes
847 ConfirmGc=Yes
848 CreateDNSDelegation=No
849 DatabasePath="C:\Windows\NTDS"
850 LogPath="C:\Windows\NTDS"
851 SYSVOLPath="C:\Windows\SYSVOL"
852 ; Set SafeModeAdminPassword to the correct value prior to using the unattend file
853 SafeModeAdminPassword=${WIN_PASS}
854 ; Run-time flags (optional)
855 RebootOnCompletion=No
856 \1a
857 ''')
858         child.expect("copied.")
859         child.expect("C:")
860         child.expect("C:")
861         child.sendline("dcpromo /answer:answers.txt")
862         i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:", pexpect.TIMEOUT], timeout=240)
863         if i == 1 or i == 2:
864             raise Exception("dcpromo failed")
865         if i == 4:  # timeout
866             child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}")
867
868         child.sendline("shutdown -r -t 0")
869         self.port_wait("${WIN_IP}", 139, wait_for_fail=True)
870         self.port_wait("${WIN_IP}", 139)
871
872         child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}")
873         # Check if we became a DC by now
874         if not self.get_is_dc(child):
875             raise Exception("dcpromo failed (and wasn't a DC even after rebooting)")
876         # Give DNS registration a kick
877         child.sendline("ipconfig /registerdns")
878
879         self.retry_cmd("host -t SRV _ldap._tcp.${WIN_REALM} ${WIN_IP}", ['has SRV record'], retries=60, delay=5)
880
881     def start_winvm(self, vm):
882         '''start a Windows VM'''
883         self.setwinvars(vm)
884
885         self.info("Joining a windows box to the domain")
886         self.vm_poweroff("${WIN_VM}", checkfail=False)
887         self.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
888
889     def run_winjoin(self, vm, domain, username="administrator", password="${PASSWORD1}"):
890         '''join a windows box to a domain'''
891         child = self.open_telnet("${WIN_HOSTNAME}", "${WIN_USER}", "${WIN_PASS}", set_time=True, set_ip=True, set_noexpire=True)
892         retries = 5
893         while retries > 0:
894             child.sendline("ipconfig /flushdns")
895             child.expect("C:")
896             child.sendline("netdom join ${WIN_HOSTNAME} /Domain:%s /UserD:%s /PasswordD:%s" % (domain, username, password))
897             i = child.expect(["The command completed successfully",
898                               "The specified domain either does not exist or could not be contacted."], timeout=120)
899             if i == 0:
900                 break
901             time.sleep(10)
902             retries -= 1
903
904         child.expect("C:")
905         child.sendline("shutdown /r -t 0")
906         self.wait_reboot()
907         child = self.open_telnet("${WIN_HOSTNAME}", "${WIN_USER}", "${WIN_PASS}", set_time=True, set_ip=True)
908         child.sendline("ipconfig /registerdns")
909         child.expect("Registration of the DNS resource records for all adapters of this computer has been initiated. Any errors will be reported in the Event Viewer")
910         child.expect("C:")
911
912     def test_remote_smbclient(self, vm, username="${WIN_USER}", password="${WIN_PASS}", args=""):
913         '''test smbclient against remote server'''
914         self.setwinvars(vm)
915         self.info('Testing smbclient')
916         self.chdir('${PREFIX}')
917         smbclient = self.getvar("smbclient")
918         self.cmd_contains("%s --version" % (smbclient), ["${SAMBA_VERSION}"])
919         self.retry_cmd('%s -L ${WIN_HOSTNAME} -U%s%%%s %s' % (smbclient, username, password, args), ["IPC"], retries=60, delay=5)
920
921     def test_net_use(self, vm, realm, domain, username, password):
922         self.setwinvars(vm)
923         self.info('Testing net use against Samba3 member')
924         child = self.open_telnet("${WIN_HOSTNAME}", "%s\\%s" % (domain, username), password)
925         child.sendline("net use t: \\\\${HOSTNAME}.%s\\test" % realm)
926         child.expect("The command completed successfully")
927
928     def setup(self, testname, subdir):
929         '''setup for main tests, parsing command line'''
930         self.parser.add_option("--conf", type='string', default='', help='config file')
931         self.parser.add_option("--skip", type='string', default='', help='list of steps to skip (comma separated)')
932         self.parser.add_option("--vms", type='string', default=None, help='list of VMs to use (comma separated)')
933         self.parser.add_option("--list", action='store_true', default=False, help='list the available steps')
934         self.parser.add_option("--rebase", action='store_true', default=False, help='do a git pull --rebase')
935         self.parser.add_option("--clean", action='store_true', default=False, help='clean the tree')
936         self.parser.add_option("--prefix", type='string', default=None, help='override install prefix')
937         self.parser.add_option("--sourcetree", type='string', default=None, help='override sourcetree location')
938         self.parser.add_option("--nocleanup", action='store_true', default=False, help='disable cleanup code')
939         self.parser.add_option("--use-ntvfs", action='store_true', default=False, help='use NTVFS for the fileserver')
940         self.parser.add_option("--dns-backend", type="choice",
941                                choices=["SAMBA_INTERNAL", "BIND9_FLATFILE", "BIND9_DLZ", "NONE"],
942                                help="The DNS server backend. SAMBA_INTERNAL is the builtin name server (default), "
943                                "BIND9_FLATFILE uses bind9 text database to store zone information, "
944                                "BIND9_DLZ uses samba4 AD to store zone information, "
945                                "NONE skips the DNS setup entirely (not recommended)",
946                                default="SAMBA_INTERNAL")
947
948         self.opts, self.args = self.parser.parse_args()
949
950         if not self.opts.conf:
951             print("Please specify a config file with --conf")
952             sys.exit(1)
953
954         # we don't need fsync safety in these tests
955         self.putenv('TDB_NO_FSYNC', '1')
956
957         self.load_config(self.opts.conf)
958
959         nameserver = self.get_nameserver()
960         if nameserver == self.getvar('NAMED_INTERFACE_IP'):
961             raise RuntimeError("old /etc/resolv.conf must not contain %s as a nameserver, this will create loops with the generated dns configuration" % nameserver)
962         self.setvar('DNSSERVER', nameserver)
963
964         self.set_skip(self.opts.skip)
965         self.set_vms(self.opts.vms)
966
967         if self.opts.list:
968             self.list_steps_mode()
969
970         if self.opts.prefix:
971             self.setvar('PREFIX', self.opts.prefix)
972
973         if self.opts.sourcetree:
974             self.setvar('SOURCETREE', self.opts.sourcetree)
975
976         if self.opts.rebase:
977             self.info('rebasing')
978             self.chdir('${SOURCETREE}')
979             self.run_cmd('git pull --rebase')
980
981         if self.opts.clean:
982             self.info('cleaning')
983             self.chdir('${SOURCETREE}/' + subdir)
984             self.run_cmd('make clean')
985
986         if self.opts.use_ntvfs:
987             self.setvar('USE_NTVFS', "--use-ntvfs")
988         else:
989             self.setvar('USE_NTVFS', "")
990
991         self.setvar('NAMESERVER_BACKEND', self.opts.dns_backend)
992
993         self.setvar('DNS_FORWARDER', "--option=dns forwarder=%s" % nameserver)