PY3: change shebang to python3 in wintest dir
[amitay/samba.git] / wintest / wintest.py
index b58cff597daf565ddc298224839e34ed2ed2f910..c239bab5506cc68c1cf25ca28560460bd77e6adb 100644 (file)
@@ -1,10 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 '''automated testing library for testing Samba against windows'''
 
-import pexpect, subprocess
+import pexpect
+import subprocess
 import optparse
-import sys, os, time, re
+import sys
+import os
+import time
+import re
+
 
 class wintest():
     '''testing of Samba against windows VMs'''
@@ -26,6 +31,11 @@ class wintest():
             self.run_cmd('ifconfig ${INTERFACE} inet6 del ${INTERFACE_IPV6}/64', checkfail=False)
             self.run_cmd('ifconfig ${INTERFACE} inet6 add ${INTERFACE_IPV6}/64 up')
 
+        self.run_cmd('ifconfig ${NAMED_INTERFACE} ${NAMED_INTERFACE_NET} up')
+        if self.getvar('NAMED_INTERFACE_IPV6'):
+            self.run_cmd('ifconfig ${NAMED_INTERFACE} inet6 del ${NAMED_INTERFACE_IPV6}/64', checkfail=False)
+            self.run_cmd('ifconfig ${NAMED_INTERFACE} inet6 add ${NAMED_INTERFACE_IPV6}/64 up')
+
     def stop_vms(self):
         '''Shut down any existing alive VMs, so they do not collide with what we are doing'''
         self.info('Shutting down any of our VMs already running')
@@ -39,7 +49,7 @@ class wintest():
 
     def getvar(self, varname):
         '''return a substitution variable'''
-        if not varname in self.vars:
+        if varname not in self.vars:
             return None
         return self.vars[varname]
 
@@ -48,9 +58,9 @@ class wintest():
         for v in ['VM', 'HOSTNAME', 'USER', 'PASS', 'SNAPSHOT', 'REALM', 'DOMAIN', 'IP']:
             vname = '%s_%s' % (vm, v)
             if vname in self.vars:
-                self.setvar("%s_%s" % (prefix,v), self.substitute("${%s}" % vname))
+                self.setvar("%s_%s" % (prefix, v), self.substitute("${%s}" % vname))
             else:
-                self.vars.pop("%s_%s" % (prefix,v), None)
+                self.vars.pop("%s_%s" % (prefix, v), None)
 
         if self.getvar("WIN_REALM"):
             self.setvar("WIN_REALM", self.getvar("WIN_REALM").upper())
@@ -76,7 +86,7 @@ class wintest():
             if colon == -1:
                 raise RuntimeError("Invalid config line '%s'" % line)
             varname = line[0:colon].strip()
-            value   = line[colon+1:].strip()
+            value   = line[colon + 1:].strip()
             self.setvar(varname, value)
 
     def list_steps_mode(self):
@@ -122,8 +132,8 @@ class wintest():
             var_end = text.find("}", var_start)
             if var_end == -1:
                 return text
-            var_name = text[var_start+2:var_end]
-            if not var_name in self.vars:
+            var_name = text[var_start + 2:var_end]
+            if var_name not in self.vars:
                 raise RuntimeError("Unknown substitution variable ${%s}" % var_name)
             text = text.replace("${%s}" % var_name, self.vars[var_name])
         return text
@@ -169,15 +179,14 @@ class wintest():
         if output:
             return subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=dir).communicate()[0]
         if isinstance(cmd, list):
-            shell=False
+            shell = False
         else:
-            shell=True
+            shell = True
         if checkfail:
             return subprocess.check_call(cmd, shell=shell, cwd=dir)
         else:
             return subprocess.call(cmd, shell=shell, cwd=dir)
 
-
     def run_child(self, cmd, dir="."):
         '''create a child and return the Popen handle to it'''
         cwd = os.getcwd()
@@ -187,9 +196,9 @@ class wintest():
         else:
             self.info('$ ' + cmd)
         if isinstance(cmd, list):
-            shell=False
+            shell = False
         else:
-            shell=True
+            shell = True
         os.chdir(dir)
         ret = subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT)
         os.chdir(cwd)
@@ -297,7 +306,7 @@ class wintest():
     def named_supports_gssapi_keytab(self):
         '''see if named supports tkey-gssapi-keytab'''
         self.write_file("${PREFIX}/named.conf.test",
-                     'options { tkey-gssapi-keytab "test"; };')
+                        'options { tkey-gssapi-keytab "test"; };')
         try:
             self.run_cmd("${NAMED_CHECKCONF} ${PREFIX}/named.conf.test")
         except subprocess.CalledProcessError:
@@ -320,38 +329,35 @@ nameserver %s
         self.write_file('/etc/resolv.conf.wintest', contents, mode='a')
         self.write_file('/etc/resolv.conf.wintest-bak', contents)
         self.run_cmd("mv -f /etc/resolv.conf.wintest /etc/resolv.conf")
-        self.resolv_conf_backup = '/etc/resolv.conf.wintest-bak';
+        self.resolv_conf_backup = '/etc/resolv.conf.wintest-bak'
 
     def configure_bind(self, kerberos_support=False, include=None):
         self.chdir('${PREFIX}')
 
-        nameserver = self.get_nameserver()
-        if nameserver == self.getvar('INTERFACE_IP'):
-            raise RuntimeError("old /etc/resolv.conf must not contain %s as a nameserver, this will create loops with the generated dns configuration" % nameserver)
-        self.setvar('DNSSERVER', nameserver)
-
-        if self.getvar('INTERFACE_IPV6'):
-            ipv6_listen = 'listen-on-v6 port 53 { ${INTERFACE_IPV6}; };'
+        if self.getvar('NAMED_INTERFACE_IPV6'):
+            ipv6_listen = 'listen-on-v6 port 53 { ${NAMED_INTERFACE_IPV6}; };'
         else:
             ipv6_listen = ''
         self.setvar('BIND_LISTEN_IPV6', ipv6_listen)
 
         if not kerberos_support:
             self.setvar("NAMED_TKEY_OPTION", "")
-        else:
+        elif self.getvar('NAMESERVER_BACKEND') != 'SAMBA_INTERNAL':
             if self.named_supports_gssapi_keytab():
                 self.setvar("NAMED_TKEY_OPTION",
-                         'tkey-gssapi-keytab "${PREFIX}/private/dns.keytab";')
+                            'tkey-gssapi-keytab "${PREFIX}/bind-dns/dns.keytab";')
             else:
                 self.info("LCREALM=${LCREALM}")
                 self.setvar("NAMED_TKEY_OPTION",
-                         '''tkey-gssapi-credential "DNS/${LCREALM}";
+                            '''tkey-gssapi-credential "DNS/${LCREALM}";
                             tkey-domain "${LCREALM}";
                  ''')
-            self.putenv('KEYTAB_FILE', '${PREFIX}/private/dns.keytab')
-            self.putenv('KRB5_KTNAME', '${PREFIX}/private/dns.keytab')
+            self.putenv('KEYTAB_FILE', '${PREFIX}/bind-dns/dns.keytab')
+            self.putenv('KRB5_KTNAME', '${PREFIX}/bind-dns/dns.keytab')
+        else:
+            self.setvar("NAMED_TKEY_OPTION", "")
 
-        if include:
+        if include and self.getvar('NAMESERVER_BACKEND') != 'SAMBA_INTERNAL':
             self.setvar("NAMED_INCLUDE", 'include "%s";' % include)
         else:
             self.setvar("NAMED_INCLUDE", '')
@@ -360,7 +366,7 @@ nameserver %s
 
         self.write_file("etc/named.conf", '''
 options {
-       listen-on port 53 { ${INTERFACE_IP};  };
+       listen-on port 53 { ${NAMED_INTERFACE_IP};  };
        ${BIND_LISTEN_IPV6}
        directory       "${PREFIX}/var/named";
        dump-file       "${PREFIX}/var/named/data/cache_dump.db";
@@ -386,18 +392,32 @@ key "rndc-key" {
 };
 
 controls {
-       inet ${INTERFACE_IP} port 953
+       inet ${NAMED_INTERFACE_IP} port 953
        allow { any; } keys { "rndc-key"; };
 };
 
 ${NAMED_INCLUDE}
 ''')
 
+        if self.getvar('NAMESERVER_BACKEND') == 'SAMBA_INTERNAL':
+            self.write_file('etc/named.conf',
+                            '''
+zone "%s" IN {
+      type forward;
+      forward only;
+      forwarders {
+         %s;
+      };
+};
+''' % (self.getvar('LCREALM'), self.getvar('INTERFACE_IP')),
+                   mode='a')
+
         # add forwarding for the windows domains
         domains = self.get_domains()
+
         for d in domains:
             self.write_file('etc/named.conf',
-                         '''
+                            '''
 zone "%s" IN {
       type forward;
       forward only;
@@ -408,7 +428,6 @@ zone "%s" IN {
 ''' % (d, domains[d]),
                      mode='a')
 
-
         self.write_file("etc/rndc.conf", '''
 # Start of rndc.conf
 key "rndc-key" {
@@ -418,33 +437,31 @@ key "rndc-key" {
 
 options {
        default-key "rndc-key";
-       default-server  ${INTERFACE_IP};
+       default-server  ${NAMED_INTERFACE_IP};
        default-port 953;
 };
 ''')
 
-
     def stop_bind(self):
         '''Stop our private BIND from listening and operating'''
         self.rndc_cmd("stop", checkfail=False)
-        self.port_wait("${INTERFACE_IP}", 53, wait_for_fail=True)
+        self.port_wait("${NAMED_INTERFACE_IP}", 53, wait_for_fail=True)
 
         self.run_cmd("rm -rf var/named")
 
-
     def start_bind(self):
         '''restart the test environment version of bind'''
         self.info("Restarting bind9")
         self.chdir('${PREFIX}')
 
-        self.set_nameserver(self.getvar('INTERFACE_IP'))
+        self.set_nameserver(self.getvar('NAMED_INTERFACE_IP'))
 
         self.run_cmd("mkdir -p var/named/data")
         self.run_cmd("chown -R ${BIND_USER} var/named")
 
         self.bind_child = self.run_child("${BIND9} -u ${BIND_USER} -n 1 -c ${PREFIX}/etc/named.conf -g")
 
-        self.port_wait("${INTERFACE_IP}", 53)
+        self.port_wait("${NAMED_INTERFACE_IP}", 53)
         self.rndc_cmd("flush")
 
     def restart_bind(self, kerberos_support=False, include=None):
@@ -458,7 +475,6 @@ options {
             self.info("restoring /etc/resolv.conf")
             self.run_cmd("mv -f %s /etc/resolv.conf" % self.resolv_conf_backup)
 
-
     def vm_poweroff(self, vmname, checkfail=True):
         '''power off a VM'''
         self.setvar('VMNAME', vmname)
@@ -478,7 +494,7 @@ options {
     def ping_wait(self, hostname):
         '''wait for a hostname to come up on the network'''
         hostname = self.substitute(hostname)
-        loops=10
+        loops = 10
         while loops > 0:
             try:
                 self.run_cmd("ping -c 1 -w 10 %s" % hostname)
@@ -491,8 +507,25 @@ options {
 
     def port_wait(self, hostname, port, retries=200, delay=3, wait_for_fail=False):
         '''wait for a host to come up on the network'''
-        self.retry_cmd("nc -v -z -w 1 %s %u" % (hostname, port), ['succeeded'],
-                       retries=retries, delay=delay, wait_for_fail=wait_for_fail)
+
+        while retries > 0:
+            child = self.pexpect_spawn("nc -v -z -w 1 %s %u" % (hostname, port), crlf=False, timeout=1)
+            child.expect([pexpect.EOF, pexpect.TIMEOUT])
+            child.close()
+            i = child.exitstatus
+            if wait_for_fail:
+                # wait for timeout or fail
+                if i is None or i > 0:
+                    return
+            else:
+                if i == 0:
+                    return
+
+            time.sleep(delay)
+            retries -= 1
+            self.info("retrying (retries=%u delay=%u)" % (retries, delay))
+
+        raise RuntimeError("gave up waiting for %s:%d" % (hostname, port))
 
     def run_net_time(self, child):
         '''run net time on windows'''
@@ -531,14 +564,15 @@ options {
         child.expect('\d+.\d+.\d+.\d+')
         self.setvar('WIN_SUBNET_MASK', child.after)
         child.expect('Default Gateway')
-        child.expect('\d+.\d+.\d+.\d+')
-        self.setvar('WIN_DEFAULT_GATEWAY', child.after)
-        child.expect("C:")
+        i = child.expect(['\d+.\d+.\d+.\d+', "C:"])
+        if i == 0:
+            self.setvar('WIN_DEFAULT_GATEWAY', child.after)
+            child.expect("C:")
 
     def get_is_dc(self, child):
         '''check if a windows machine is a domain controller'''
         child.sendline("dcdiag")
-        i = child.expect(["is not a Directory Server",
+        i = child.expect(["is not a [Directory Server|DC]",
                           "is not recognized as an internal or external command",
                           "Home Server = ",
                           "passed test Replications"])
@@ -587,7 +621,7 @@ options {
             child.expect("C:")
 
     def set_dns(self, child):
-        child.sendline('netsh interface ip set dns "${WIN_NIC}" static ${INTERFACE_IP} primary')
+        child.sendline('netsh interface ip set dns "${WIN_NIC}" static ${NAMED_INTERFACE_IP} primary')
         i = child.expect(['C:', pexpect.EOF, pexpect.TIMEOUT], timeout=5)
         if i > 0:
             return True
@@ -620,7 +654,6 @@ options {
         child.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=5)
         return True
 
-
     def resolve_ip(self, hostname, retries=60, delay=5):
         '''resolve an IP given a hostname, assuming NBT'''
         while retries > 0:
@@ -637,12 +670,13 @@ options {
             self.info("retrying (retries=%u delay=%u)" % (retries, delay))
         raise RuntimeError("Failed to resolve IP of %s" % hostname)
 
-
     def open_telnet(self, hostname, username, password, retries=60, delay=5, set_time=False, set_ip=False,
                     disable_firewall=True, run_tlntadmn=True, set_noexpire=False):
         '''open a telnet connection to a windows server, return the pexpect child'''
         set_route = False
         set_dns = False
+        set_telnetclients = True
+        start_telnet = True
         if self.getvar('WIN_IP'):
             ip = self.getvar('WIN_IP')
         else:
@@ -666,12 +700,34 @@ options {
             child.expect("password:")
             child.sendline(password)
             i = child.expect(["C:",
+                              "TelnetClients",
                               "Denying new connections due to the limit on number of connections",
                               "No more connections are allowed to telnet server",
                               "Unable to connect to remote host",
                               "No route to host",
                               "Connection refused",
                               pexpect.EOF])
+            if i == 1:
+                if set_telnetclients:
+                    self.run_cmd('bin/net rpc group add TelnetClients -S $WIN_IP -U$WIN_USER%$WIN_PASS')
+                    self.run_cmd('bin/net rpc group addmem TelnetClients "authenticated users" -S $WIN_IP -U$WIN_USER%$WIN_PASS')
+                    child.close()
+                    retries -= 1
+                    set_telnetclients = False
+                    self.info("retrying (retries=%u delay=%u)" % (retries, delay))
+                    continue
+                else:
+                    raise RuntimeError("Failed to connect with telnet due to missing TelnetClients membership")
+
+            if i == 6:
+                # This only works if it is installed and enabled, but not started.  Not entirely likely, but possible
+                self.run_cmd('bin/net rpc service start TlntSvr -S $WIN_IP -U$WIN_USER%$WIN_PASS')
+                child.close()
+                start_telnet = False
+                retries -= 1
+                self.info("retrying (retries=%u delay=%u)" % (retries, delay))
+                continue
+
             if i != 0:
                 child.close()
                 time.sleep(delay)
@@ -681,7 +737,7 @@ options {
             if set_dns:
                 set_dns = False
                 if self.set_dns(child):
-                    continue;
+                    continue
             if set_route:
                 child.sendline('route add 0.0.0.0 mask 0.0.0.0 ${WIN_DEFAULT_GATEWAY}')
                 child.expect("C:")
@@ -760,7 +816,6 @@ options {
                 ret.append(self.vars[v])
         return ret
 
-
     def run_dcpromo_as_first_dc(self, vm, func_level=None):
         self.setwinvars(vm)
         self.info("Configuring a windows VM ${WIN_VM} at the first DC in the domain using dcpromo")
@@ -804,19 +859,29 @@ RebootOnCompletion=No
         child.expect("C:")
         child.expect("C:")
         child.sendline("dcpromo /answer:answers.txt")
-        i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:"], timeout=240)
+        i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:", pexpect.TIMEOUT], timeout=240)
         if i == 1 or i == 2:
             raise Exception("dcpromo failed")
+        if i == 4:  # timeout
+            child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}")
+
         child.sendline("shutdown -r -t 0")
         self.port_wait("${WIN_IP}", 139, wait_for_fail=True)
         self.port_wait("${WIN_IP}", 139)
-        self.retry_cmd("host -t SRV _ldap._tcp.${WIN_REALM} ${WIN_IP}", ['has SRV record'] )
 
+        child = self.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}")
+        # Check if we became a DC by now
+        if not self.get_is_dc(child):
+            raise Exception("dcpromo failed (and wasn't a DC even after rebooting)")
+        # Give DNS registration a kick
+        child.sendline("ipconfig /registerdns")
+
+        self.retry_cmd("host -t SRV _ldap._tcp.${WIN_REALM} ${WIN_IP}", ['has SRV record'], retries=60, delay=5)
 
     def start_winvm(self, vm):
         '''start a Windows VM'''
         self.setwinvars(vm)
-        
+
         self.info("Joining a windows box to the domain")
         self.vm_poweroff("${WIN_VM}", checkfail=False)
         self.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
@@ -829,8 +894,8 @@ RebootOnCompletion=No
             child.sendline("ipconfig /flushdns")
             child.expect("C:")
             child.sendline("netdom join ${WIN_HOSTNAME} /Domain:%s /UserD:%s /PasswordD:%s" % (domain, username, password))
-            i = child.expect(["The command completed successfully", 
-                             "The specified domain either does not exist or could not be contacted."])
+            i = child.expect(["The command completed successfully",
+                              "The specified domain either does not exist or could not be contacted."], timeout=120)
             if i == 0:
                 break
             time.sleep(10)
@@ -844,14 +909,14 @@ RebootOnCompletion=No
         child.expect("Registration of the DNS resource records for all adapters of this computer has been initiated. Any errors will be reported in the Event Viewer")
         child.expect("C:")
 
-
     def test_remote_smbclient(self, vm, username="${WIN_USER}", password="${WIN_PASS}", args=""):
         '''test smbclient against remote server'''
         self.setwinvars(vm)
         self.info('Testing smbclient')
         self.chdir('${PREFIX}')
-        self.cmd_contains("bin/smbclient --version", ["${SAMBA_VERSION}"])
-        self.retry_cmd('bin/smbclient -L ${WIN_HOSTNAME} -U%s%%%s %s' % (username, password, args), ["IPC"])
+        smbclient = self.getvar("smbclient")
+        self.cmd_contains("%s --version" % (smbclient), ["${SAMBA_VERSION}"])
+        self.retry_cmd('%s -L ${WIN_HOSTNAME} -U%s%%%s %s' % (smbclient, username, password, args), ["IPC"], retries=60, delay=5)
 
     def test_net_use(self, vm, realm, domain, username, password):
         self.setwinvars(vm)
@@ -860,7 +925,6 @@ RebootOnCompletion=No
         child.sendline("net use t: \\\\${HOSTNAME}.%s\\test" % realm)
         child.expect("The command completed successfully")
 
-
     def setup(self, testname, subdir):
         '''setup for main tests, parsing command line'''
         self.parser.add_option("--conf", type='string', default='', help='config file')
@@ -872,6 +936,14 @@ RebootOnCompletion=No
         self.parser.add_option("--prefix", type='string', default=None, help='override install prefix')
         self.parser.add_option("--sourcetree", type='string', default=None, help='override sourcetree location')
         self.parser.add_option("--nocleanup", action='store_true', default=False, help='disable cleanup code')
+        self.parser.add_option("--use-ntvfs", action='store_true', default=False, help='use NTVFS for the fileserver')
+        self.parser.add_option("--dns-backend", type="choice",
+                               choices=["SAMBA_INTERNAL", "BIND9_FLATFILE", "BIND9_DLZ", "NONE"],
+                               help="The DNS server backend. SAMBA_INTERNAL is the builtin name server (default), "
+                               "BIND9_FLATFILE uses bind9 text database to store zone information, "
+                               "BIND9_DLZ uses samba4 AD to store zone information, "
+                               "NONE skips the DNS setup entirely (not recommended)",
+                               default="SAMBA_INTERNAL")
 
         self.opts, self.args = self.parser.parse_args()
 
@@ -884,6 +956,11 @@ RebootOnCompletion=No
 
         self.load_config(self.opts.conf)
 
+        nameserver = self.get_nameserver()
+        if nameserver == self.getvar('NAMED_INTERFACE_IP'):
+            raise RuntimeError("old /etc/resolv.conf must not contain %s as a nameserver, this will create loops with the generated dns configuration" % nameserver)
+        self.setvar('DNSSERVER', nameserver)
+
         self.set_skip(self.opts.skip)
         self.set_vms(self.opts.vms)
 
@@ -905,3 +982,12 @@ RebootOnCompletion=No
             self.info('cleaning')
             self.chdir('${SOURCETREE}/' + subdir)
             self.run_cmd('make clean')
+
+        if self.opts.use_ntvfs:
+            self.setvar('USE_NTVFS', "--use-ntvfs")
+        else:
+            self.setvar('USE_NTVFS', "")
+
+        self.setvar('NAMESERVER_BACKEND', self.opts.dns_backend)
+
+        self.setvar('DNS_FORWARDER', "--option=dns forwarder=%s" % nameserver)