wintest/test-s4-howto.py

   1 #!/usr/bin/env python
   2
   3 '''automated testing of the steps of the Samba4 HOWTO'''
   4
   5 import sys, os
   6 import optparse
   7 import wintest, pexpect, time
   8
   9 def check_prerequesites(t):
  10     t.info("Checking prerequesites")
  11     t.setvar('HOSTNAME', t.cmd_output("hostname -s").strip())
  12     if os.getuid() != 0:
  13         raise Exception("You must run this script as root")
  14     t.putenv("KRB5_CONFIG", '${PREFIX}/private/krb5.conf')
  15     t.run_cmd('ifconfig ${INTERFACE} ${INTERFACE_NET} up')
  16     if t.getvar('INTERFACE_IPV6'):
  17         t.run_cmd('ifconfig ${INTERFACE} inet6 del ${INTERFACE_IPV6}/64', checkfail=False)
  18         t.run_cmd('ifconfig ${INTERFACE} inet6 add ${INTERFACE_IPV6}/64 up')
  19
  20 def build_s4(t):
  21     '''build samba4'''
  22     t.info('Building s4')
  23     t.chdir('${SOURCETREE}/source4')
  24     t.putenv('CC', 'ccache gcc')
  25     t.run_cmd('make reconfigure || ./configure --enable-auto-reconfigure --enable-developer --prefix=${PREFIX} -C')
  26     t.run_cmd('make -j')
  27     t.run_cmd('rm -rf ${PREFIX}')
  28     t.run_cmd('make -j install')
  29
  30
  31 def provision_s4(t, func_level="2008"):
  32     '''provision s4 as a DC'''
  33     t.info('Provisioning s4')
  34     t.chdir('${PREFIX}')
  35     t.del_files(["var", "private"])
  36     t.run_cmd("rm -f etc/smb.conf")
  37     provision=['sbin/provision',
  38                '--realm=${LCREALM}',
  39                '--domain=${DOMAIN}',
  40                '--adminpass=${PASSWORD1}',
  41                '--server-role=domain controller',
  42                '--function-level=%s' % func_level,
  43                '-d${DEBUGLEVEL}',
  44                '--option=interfaces=${INTERFACE}',
  45                '--host-ip=${INTERFACE_IP}',
  46                '--option=bind interfaces only=yes',
  47                '--option=rndc command=${RNDC} -c${PREFIX}/etc/rndc.conf']
  48     if t.getvar('INTERFACE_IPV6'):
  49         provision.append('--host-ip6=${INTERFACE_IPV6}')
  50     t.run_cmd(provision)
  51     t.run_cmd('bin/samba-tool newuser testallowed ${PASSWORD1}')
  52     t.run_cmd('bin/samba-tool newuser testdenied ${PASSWORD1}')
  53     t.run_cmd('bin/samba-tool group addmembers "Allowed RODC Password Replication Group" testallowed')
  54
  55
  56 def start_s4(t):
  57     '''startup samba4'''
  58     t.info('Starting Samba4')
  59     t.chdir("${PREFIX}")
  60     t.run_cmd('killall -9 -q samba smbd nmbd winbindd', checkfail=False)
  61     t.run_cmd(['sbin/samba',
  62              '--option', 'panic action=gnome-terminal -e "gdb --pid %PID%"'])
  63     t.port_wait("${INTERFACE_IP}", 139)
  64
  65 def stop_vms(t):
  66     '''Shut down any existing alive VMs, so they do not collide with what we are doing'''
  67     t.info('Shutting down any of our VMs already running')
  68     vms = t.get_vms()
  69     for v in vms:
  70         t.vm_poweroff(v, checkfail=False)
  71
  72 def test_smbclient(t):
  73     '''test smbclient'''
  74     t.info('Testing smbclient')
  75     t.chdir('${PREFIX}')
  76     t.cmd_contains("bin/smbclient --version", ["Version 4.0"])
  77     t.retry_cmd('bin/smbclient -L ${INTERFACE_IP} -U%', ["netlogon", "sysvol", "IPC Service"])
  78     child = t.pexpect_spawn('bin/smbclient //${INTERFACE_IP}/netlogon -Uadministrator%${PASSWORD1}')
  79     child.expect("smb:")
  80     child.sendline("dir")
  81     child.expect("blocks available")
  82     child.sendline("mkdir testdir")
  83     child.expect("smb:")
  84     child.sendline("cd testdir")
  85     child.expect('testdir')
  86     child.sendline("cd ..")
  87     child.sendline("rmdir testdir")
  88
  89
  90 def create_shares(t):
  91     '''create some test shares'''
  92     t.info("Adding test shares")
  93     t.chdir('${PREFIX}')
  94     t.write_file("etc/smb.conf", '''
  95 [test]
  96        path = ${PREFIX}/test
  97        read only = no
  98 [profiles]
  99        path = ${PREFIX}/var/profiles
 100        read only = no
 101     ''',
 102                  mode='a')
 103     t.run_cmd("mkdir -p test")
 104     t.run_cmd("mkdir -p var/profiles")
 105
 106
 107 def set_nameserver(t, nameserver):
 108     '''set the nameserver in resolv.conf'''
 109     t.write_file("/etc/resolv.conf.wintest", '''
 110 # Generated by wintest, the Samba v Windows automated testing system
 111 nameserver %s
 112
 113 # your original resolv.conf appears below:
 114 ''' % t.substitute(nameserver))
 115     child = t.pexpect_spawn("cat /etc/resolv.conf", crlf=False)
 116     i = child.expect(['your original resolv.conf appears below:', pexpect.EOF])
 117     if i == 0:
 118         child.expect(pexpect.EOF)
 119     contents = child.before.lstrip().replace('\r', '')
 120     t.write_file('/etc/resolv.conf.wintest', contents, mode='a')
 121     t.write_file('/etc/resolv.conf.wintest-bak', contents)
 122     t.run_cmd("mv -f /etc/resolv.conf.wintest /etc/resolv.conf")
 123     t.resolv_conf_backup = '/etc/resolv.conf.wintest-bak';
 124
 125
 126 def restore_resolv_conf(t):
 127     '''restore the /etc/resolv.conf after testing is complete'''
 128     if getattr(t, 'resolv_conf_backup', False):
 129         t.info("restoring /etc/resolv.conf")
 130         t.run_cmd("mv -f %s /etc/resolv.conf" % t.resolv_conf_backup)
 131
 132
 133 def rndc_cmd(t, cmd, checkfail=True):
 134     '''run a rndc command'''
 135     t.run_cmd("${RNDC} -c ${PREFIX}/etc/rndc.conf %s" % cmd, checkfail=checkfail)
 136
 137 def configure_bind(t):
 138     t.chdir('${PREFIX}')
 139
 140     nameserver = t.get_nameserver()
 141     if nameserver == t.getvar('INTERFACE_IP'):
 142         raise RuntimeError("old /etc/resolv.conf must not contain %s as a nameserver, this will create loops with the generated dns configuration" % nameserver)
 143     t.setvar('DNSSERVER', nameserver)
 144
 145     if t.getvar('INTERFACE_IPV6'):
 146         ipv6_listen = 'listen-on-v6 port 53 { ${INTERFACE_IPV6}; };'
 147     else:
 148         ipv6_listen = ''
 149     t.setvar('BIND_LISTEN_IPV6', ipv6_listen)
 150
 151     t.write_file("etc/named.conf", '''
 152 options {
 153         listen-on port 53 { ${INTERFACE_IP};  };
 154         ${BIND_LISTEN_IPV6}
 155         directory       "${PREFIX}/var/named";
 156         dump-file       "${PREFIX}/var/named/data/cache_dump.db";
 157         pid-file        "${PREFIX}/var/named/named.pid";
 158         statistics-file "${PREFIX}/var/named/data/named_stats.txt";
 159         memstatistics-file "${PREFIX}/var/named/data/named_mem_stats.txt";
 160         allow-query     { any; };
 161         recursion yes;
 162         tkey-gssapi-credential "DNS/${HOSTNAME}.${LCREALM}";
 163         tkey-domain "${REALM}";
 164         max-cache-ttl 10;
 165         max-ncache-ttl 10;
 166
 167         forward only;
 168         forwarders {
 169                   ${DNSSERVER};
 170         };
 171
 172 };
 173
 174 key "rndc-key" {
 175         algorithm hmac-md5;
 176         secret "lA/cTrno03mt5Ju17ybEYw==";
 177 };
 178
 179 controls {
 180         inet ${INTERFACE_IP} port 953
 181         allow { any; } keys { "rndc-key"; };
 182 };
 183
 184 include "${PREFIX}/private/named.conf";
 185 ''')
 186
 187     # add forwarding for the windows domains
 188     domains = t.get_domains()
 189     for d in domains:
 190         t.write_file('etc/named.conf',
 191                      '''
 192 zone "%s" IN {
 193       type forward;
 194       forward only;
 195       forwarders {
 196          %s;
 197       };
 198 };
 199 ''' % (d, domains[d]),
 200                      mode='a')
 201
 202
 203     t.write_file("etc/rndc.conf", '''
 204 # Start of rndc.conf
 205 key "rndc-key" {
 206         algorithm hmac-md5;
 207         secret "lA/cTrno03mt5Ju17ybEYw==";
 208 };
 209
 210 options {
 211         default-key "rndc-key";
 212         default-server  ${INTERFACE_IP};
 213         default-port 953;
 214 };
 215 ''')
 216
 217     set_nameserver(t, t.getvar('INTERFACE_IP'))
 218
 219
 220 def stop_bind(t):
 221     '''Stop our private BIND from listening and operating'''
 222     rndc_cmd(t, "stop", checkfail=False)
 223     t.port_wait("${INTERFACE_IP}", 53, wait_for_fail=True)
 224
 225     t.run_cmd("rm -rf var/named")
 226
 227
 228 def start_bind(t):
 229     '''restart the test environment version of bind'''
 230     t.info("Restarting bind9")
 231     t.putenv('KEYTAB_FILE', '${PREFIX}/private/dns.keytab')
 232     t.putenv('KRB5_KTNAME', '${PREFIX}/private/dns.keytab')
 233     t.chdir('${PREFIX}')
 234
 235     set_nameserver(t, t.getvar('INTERFACE_IP'))
 236
 237     t.run_cmd("mkdir -p var/named/data")
 238     t.run_cmd("chown -R ${BIND_USER} var/named")
 239
 240     t.bind_child = t.run_child("${BIND9} -u ${BIND_USER} -n 1 -c ${PREFIX}/etc/named.conf -g")
 241
 242     t.port_wait("${INTERFACE_IP}", 53)
 243     rndc_cmd(t, "flush")
 244
 245 def restart_bind(t):
 246     configure_bind(t)
 247     stop_bind(t)
 248     start_bind(t)
 249
 250 def test_dns(t):
 251     '''test that DNS is OK'''
 252     t.info("Testing DNS")
 253     t.cmd_contains("host -t SRV _ldap._tcp.${LCREALM}.",
 254                  ['_ldap._tcp.${LCREALM} has SRV record 0 100 389 ${HOSTNAME}.${LCREALM}'])
 255     t.cmd_contains("host -t SRV  _kerberos._udp.${LCREALM}.",
 256                  ['_kerberos._udp.${LCREALM} has SRV record 0 100 88 ${HOSTNAME}.${LCREALM}'])
 257     t.cmd_contains("host -t A ${HOSTNAME}.${LCREALM}",
 258                  ['${HOSTNAME}.${LCREALM} has address'])
 259
 260 def test_kerberos(t):
 261     '''test that kerberos is OK'''
 262     t.info("Testing kerberos")
 263     t.run_cmd("kdestroy")
 264     t.kinit("administrator@${REALM}", "${PASSWORD1}")
 265     # this copes with the differences between MIT and Heimdal klist
 266     t.cmd_contains("klist", ["rincipal", "administrator@${REALM}"])
 267
 268
 269 def test_dyndns(t):
 270     '''test that dynamic DNS is working'''
 271     t.chdir('${PREFIX}')
 272     t.run_cmd("sbin/samba_dnsupdate --fail-immediately")
 273     rndc_cmd(t, "flush")
 274
 275
 276 def run_winjoin(t, vm):
 277     '''join a windows box to our domain'''
 278     t.setwinvars(vm)
 279
 280     t.info("Joining a windows box to the domain")
 281     t.vm_poweroff("${WIN_VM}", checkfail=False)
 282     t.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
 283     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_USER}", "${WIN_PASS}", set_time=True, set_ip=True)
 284     child.sendline("netdom join ${WIN_HOSTNAME} /Domain:${LCREALM} /PasswordD:${PASSWORD1} /UserD:administrator")
 285     child.expect("The command completed successfully")
 286     child.expect("C:")
 287     child.sendline("shutdown /r -t 0")
 288     t.wait_reboot()
 289     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_USER}", "${WIN_PASS}", set_time=True, set_ip=True)
 290     child.sendline("ipconfig /registerdns")
 291     child.expect("Registration of the DNS resource records for all adapters of this computer has been initiated. Any errors will be reported in the Event Viewer")
 292     child.expect("C:")
 293
 294 def test_winjoin(t, vm):
 295     t.info("Checking the windows join is OK")
 296     t.chdir('${PREFIX}')
 297     t.port_wait("${WIN_IP}", 139)
 298     t.retry_cmd('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Uadministrator@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"], retries=100)
 299     t.cmd_contains("host -t A ${WIN_HOSTNAME}.${LCREALM}.", ['has address'])
 300     t.cmd_contains('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utestallowed@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 301     t.cmd_contains('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -k no -Utestallowed@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 302     t.cmd_contains('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -k yes -Utestallowed@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 303     child = t.open_telnet("${WIN_HOSTNAME}", "${DOMAIN}\\administrator", "${PASSWORD1}")
 304     child.sendline("net use t: \\\\${HOSTNAME}.${LCREALM}\\test")
 305     child.expect("The command completed successfully")
 306     t.vm_poweroff("${WIN_VM}")
 307
 308
 309 def run_dcpromo(t, vm):
 310     '''run a dcpromo on windows'''
 311     t.setwinvars(vm)
 312
 313     t.info("Joining a windows VM ${WIN_VM} to the domain as a DC using dcpromo")
 314     t.vm_poweroff("${WIN_VM}", checkfail=False)
 315     t.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
 316     child = t.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_ip=True)
 317     child.sendline("copy /Y con answers.txt")
 318     child.sendline('''
 319 [DCINSTALL]
 320 RebootOnSuccess=Yes
 321 RebootOnCompletion=Yes
 322 ReplicaOrNewDomain=Replica
 323 ReplicaDomainDNSName=${LCREALM}
 324 SiteName=Default-First-Site-Name
 325 InstallDNS=No
 326 ConfirmGc=Yes
 327 CreateDNSDelegation=No
 328 UserDomain=${LCREALM}
 329 UserName=${LCREALM}\\administrator
 330 Password=${PASSWORD1}
 331 DatabasePath="C:\Windows\NTDS"
 332 LogPath="C:\Windows\NTDS"
 333 SYSVOLPath="C:\Windows\SYSVOL"
 334 SafeModeAdminPassword=${PASSWORD1}
 335 \1a
 336 ''')
 337     child.expect("copied.")
 338     child.expect("C:")
 339     child.expect("C:")
 340     child.sendline("dcpromo /answer:answers.txt")
 341     i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:"], timeout=120)
 342     if i == 1 or i == 2:
 343         raise Exception("dcpromo failed")
 344     t.wait_reboot()
 345
 346
 347 def test_dcpromo(t, vm):
 348     '''test that dcpromo worked'''
 349     t.info("Checking the dcpromo join is OK")
 350     t.chdir('${PREFIX}')
 351     t.port_wait("${WIN_IP}", 139)
 352     t.retry_cmd("host -t A ${WIN_HOSTNAME}.${LCREALM}. ${INTERFACE_IP}",
 353                 ['${WIN_HOSTNAME}.${LCREALM} has address'],
 354                 retries=30, delay=10, casefold=True)
 355     t.retry_cmd('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Uadministrator@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 356     t.cmd_contains("host -t A ${WIN_HOSTNAME}.${LCREALM}.", ['has address'])
 357     t.cmd_contains('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utestallowed@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 358
 359     t.cmd_contains("bin/samba-tool drs kcc ${HOSTNAME}.${LCREALM} -Uadministrator@${LCREALM}%${PASSWORD1}", ['Consistency check', 'successful'])
 360     t.retry_cmd("bin/samba-tool drs kcc ${WIN_HOSTNAME}.${LCREALM} -Uadministrator@${LCREALM}%${PASSWORD1}", ['Consistency check', 'successful'])
 361
 362     t.kinit("administrator@${REALM}", "${PASSWORD1}")
 363
 364     # the first replication will transfer the dnsHostname attribute
 365     t.cmd_contains("bin/samba-tool drs replicate ${HOSTNAME}.${LCREALM} ${WIN_HOSTNAME} CN=Configuration,${BASEDN} -k yes", ["was successful"])
 366
 367     for nc in [ '${BASEDN}', 'CN=Configuration,${BASEDN}', 'CN=Schema,CN=Configuration,${BASEDN}' ]:
 368         t.cmd_contains("bin/samba-tool drs replicate ${HOSTNAME}.${LCREALM} ${WIN_HOSTNAME}.${LCREALM} %s -k yes" % nc, ["was successful"])
 369         t.cmd_contains("bin/samba-tool drs replicate ${WIN_HOSTNAME}.${LCREALM} ${HOSTNAME}.${LCREALM} %s -k yes" % nc, ["was successful"])
 370
 371     t.cmd_contains("bin/samba-tool drs showrepl ${HOSTNAME}.${LCREALM} -k yes",
 372                  [ "INBOUND NEIGHBORS",
 373                    "${BASEDN}",
 374                    "Last attempt .* was successful",
 375                    "CN=Configuration,${BASEDN}",
 376                    "Last attempt .* was successful",
 377                    "CN=Configuration,${BASEDN}", # cope with either order
 378                    "Last attempt .* was successful",
 379                    "OUTBOUND NEIGHBORS",
 380                    "${BASEDN}",
 381                    "Last success",
 382                    "CN=Configuration,${BASEDN}",
 383                    "Last success",
 384                    "CN=Configuration,${BASEDN}",
 385                    "Last success"],
 386                    ordered=True,
 387                    regex=True)
 388
 389     t.cmd_contains("bin/samba-tool drs showrepl ${WIN_HOSTNAME}.${LCREALM} -k yes",
 390                  [ "INBOUND NEIGHBORS",
 391                    "${BASEDN}",
 392                    "Last attempt .* was successful",
 393                    "CN=Configuration,${BASEDN}",
 394                    "Last attempt .* was successful",
 395                    "CN=Configuration,${BASEDN}",
 396                    "Last attempt .* was successful",
 397                    "OUTBOUND NEIGHBORS",
 398                    "${BASEDN}",
 399                    "Last success",
 400                    "CN=Configuration,${BASEDN}",
 401                    "Last success",
 402                    "CN=Configuration,${BASEDN}",
 403                    "Last success" ],
 404                    ordered=True,
 405                    regex=True)
 406
 407     child = t.open_telnet("${WIN_HOSTNAME}", "${DOMAIN}\\administrator", "${PASSWORD1}", set_time=True)
 408     child.sendline("net use t: \\\\${HOSTNAME}.${LCREALM}\\test")
 409
 410     retries = 10
 411     i = child.expect(["The command completed successfully", "The network path was not found"])
 412     while i == 1 and retries > 0:
 413         child.expect("C:")
 414         time.sleep(2)
 415         child.sendline("net use t: \\\\${HOSTNAME}.${LCREALM}\\test")
 416         i = child.expect(["The command completed successfully", "The network path was not found"])
 417         retries -=1
 418
 419     t.run_net_time(child)
 420
 421     t.info("Checking if showrepl is happy")
 422     child.sendline("repadmin /showrepl")
 423     child.expect("${BASEDN}")
 424     child.expect("was successful")
 425     child.expect("CN=Configuration,${BASEDN}")
 426     child.expect("was successful")
 427     child.expect("CN=Schema,CN=Configuration,${BASEDN}")
 428     child.expect("was successful")
 429
 430     t.info("Checking if new users propogate to windows")
 431     t.retry_cmd('bin/samba-tool newuser test2 ${PASSWORD2}', ["created successfully"])
 432     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k no", ['Sharename', 'Remote IPC'])
 433     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k yes", ['Sharename', 'Remote IPC'])
 434
 435     t.info("Checking if new users on windows propogate to samba")
 436     child.sendline("net user test3 ${PASSWORD3} /add")
 437     while True:
 438         i = child.expect(["The command completed successfully",
 439                           "The directory service was unable to allocate a relative identifier"])
 440         if i == 0:
 441             break
 442         time.sleep(2)
 443
 444     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${LCREALM} -Utest3%${PASSWORD3} -k no", ['Sharename', 'IPC'])
 445     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${LCREALM} -Utest3%${PASSWORD3} -k yes", ['Sharename', 'IPC'])
 446
 447     t.info("Checking propogation of user deletion")
 448     t.run_cmd('bin/samba-tool user delete test2 -Uadministrator@${LCREALM}%${PASSWORD1}')
 449     child.sendline("net user test3 /del")
 450     child.expect("The command completed successfully")
 451
 452     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k no", ['LOGON_FAILURE'])
 453     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${LCREALM} -Utest3%${PASSWORD3} -k no", ['LOGON_FAILURE'])
 454     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k yes", ['LOGON_FAILURE'])
 455     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${LCREALM} -Utest3%${PASSWORD3} -k yes", ['LOGON_FAILURE'])
 456     t.vm_poweroff("${WIN_VM}")
 457
 458
 459 def run_dcpromo_rodc(t, vm):
 460     '''run a RODC dcpromo to join a windows DC to the samba domain'''
 461     t.setwinvars(vm)
 462     t.info("Joining a w2k8 box to the domain as a RODC")
 463     t.vm_poweroff("${WIN_VM}", checkfail=False)
 464     t.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
 465     child = t.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_ip=True)
 466     child.sendline("copy /Y con answers.txt")
 467     child.sendline('''
 468 [DCInstall]
 469 ReplicaOrNewDomain=ReadOnlyReplica
 470 ReplicaDomainDNSName=${LCREALM}
 471 PasswordReplicationDenied="BUILTIN\Administrators"
 472 PasswordReplicationDenied="BUILTIN\Server Operators"
 473 PasswordReplicationDenied="BUILTIN\Backup Operators"
 474 PasswordReplicationDenied="BUILTIN\Account Operators"
 475 PasswordReplicationDenied="${DOMAIN}\Denied RODC Password Replication Group"
 476 PasswordReplicationAllowed="${DOMAIN}\Allowed RODC Password Replication Group"
 477 DelegatedAdmin="${DOMAIN}\\Administrator"
 478 SiteName=Default-First-Site-Name
 479 InstallDNS=No
 480 ConfirmGc=Yes
 481 CreateDNSDelegation=No
 482 UserDomain=${LCREALM}
 483 UserName=${LCREALM}\\administrator
 484 Password=${PASSWORD1}
 485 DatabasePath="C:\Windows\NTDS"
 486 LogPath="C:\Windows\NTDS"
 487 SYSVOLPath="C:\Windows\SYSVOL"
 488 SafeModeAdminPassword=${PASSWORD1}
 489 RebootOnCompletion=No
 490 \1a
 491 ''')
 492     child.expect("copied.")
 493     child.sendline("dcpromo /answer:answers.txt")
 494     i = child.expect(["You must restart this computer", "failed"], timeout=120)
 495     if i != 0:
 496         raise Exception("dcpromo failed")
 497     child.sendline("shutdown -r -t 0")
 498     t.wait_reboot()
 499
 500
 501
 502 def test_dcpromo_rodc(t, vm):
 503     '''test the RODC dcpromo worked'''
 504     t.info("Checking the w2k8 RODC join is OK")
 505     t.chdir('${PREFIX}')
 506     t.port_wait("${WIN_IP}", 139)
 507     t.retry_cmd('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Uadministrator@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 508     t.cmd_contains("host -t A ${WIN_HOSTNAME}.${LCREALM}.", ['has address'])
 509     t.cmd_contains('bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utestallowed@${LCREALM}%${PASSWORD1}', ["C$", "IPC$", "Sharename"])
 510     child = t.open_telnet("${WIN_HOSTNAME}", "${DOMAIN}\\administrator", "${PASSWORD1}", set_time=True)
 511     child.sendline("net use t: \\\\${HOSTNAME}.${LCREALM}\\test")
 512     child.expect("The command completed successfully")
 513
 514     t.info("Checking if showrepl is happy")
 515     child.sendline("repadmin /showrepl")
 516     child.expect("${BASEDN}")
 517     child.expect("was successful")
 518     child.expect("CN=Configuration,${BASEDN}")
 519     child.expect("was successful")
 520     child.expect("CN=Configuration,${BASEDN}")
 521     child.expect("was successful")
 522
 523     t.info("Checking if new users are available on windows")
 524     t.run_cmd('bin/samba-tool newuser test2 ${PASSWORD2}')
 525     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k yes", ['Sharename', 'Remote IPC'])
 526     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k no", ['LOGON_FAILURE'])
 527     t.retry_cmd("bin/samba-tool drs replicate ${WIN_HOSTNAME}.${LCREALM} ${HOSTNAME}.${LCREALM} ${BASEDN} -k yes", ["was successful"])
 528     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k no", ['Sharename', 'Remote IPC'])
 529     t.run_cmd('bin/samba-tool user delete test2 -Uadministrator@${LCREALM}%${PASSWORD1}')
 530     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k yes", ['LOGON_FAILURE'])
 531     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${LCREALM} -Utest2%${PASSWORD2} -k no", ['LOGON_FAILURE'])
 532     t.vm_poweroff("${WIN_VM}")
 533
 534
 535 def prep_join_as_dc(t, vm):
 536     '''start VM and shutdown Samba in preperation to join a windows domain as a DC'''
 537     t.setwinvars(vm)
 538     t.info("Starting VMs for joining ${WIN_VM} as a second DC using samba-tool join DC")
 539     t.chdir('${PREFIX}')
 540     t.run_cmd('killall -9 -q samba smbd nmbd winbindd', checkfail=False)
 541     t.vm_poweroff("${WIN_VM}", checkfail=False)
 542     t.vm_restore("${WIN_VM}", "${WIN_SNAPSHOT}")
 543     rndc_cmd(t, 'flush')
 544     t.run_cmd("rm -rf etc/smb.conf private")
 545     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_DOMAIN}\\administrator", "${WIN_PASS}", set_time=True)
 546     t.get_ipconfig(child)
 547
 548 def join_as_dc(t, vm):
 549     '''join a windows domain as a DC'''
 550     t.setwinvars(vm)
 551     t.info("Joining ${WIN_VM} as a second DC using samba-tool join DC")
 552     t.port_wait("${WIN_IP}", 389)
 553     t.retry_cmd("host -t SRV _ldap._tcp.${WIN_REALM} ${WIN_IP}", ['has SRV record'] )
 554
 555     t.retry_cmd("bin/samba-tool drs showrepl ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator%${WIN_PASS}", ['INBOUND NEIGHBORS'] )
 556     t.run_cmd('bin/samba-tool join ${WIN_REALM} DC -Uadministrator%${WIN_PASS} -d${DEBUGLEVEL} --option=interfaces=${INTERFACE}')
 557     t.run_cmd('bin/samba-tool drs kcc ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 558
 559
 560 def test_join_as_dc(t, vm):
 561     '''test the join of a windows domain as a DC'''
 562     t.info("Checking the DC join is OK")
 563     t.chdir('${PREFIX}')
 564     t.retry_cmd('bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}', ["C$", "IPC$", "Sharename"])
 565     t.cmd_contains("host -t A ${HOSTNAME}.${WIN_REALM}.", ['has address'])
 566     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_DOMAIN}\\administrator", "${WIN_PASS}", set_time=True)
 567
 568     t.info("Forcing kcc runs, and replication")
 569     t.run_cmd('bin/samba-tool drs kcc ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 570     t.run_cmd('bin/samba-tool drs kcc ${HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 571
 572     t.kinit("administrator@${WIN_REALM}", "${WIN_PASS}")
 573     for nc in [ '${WIN_BASEDN}', 'CN=Configuration,${WIN_BASEDN}', 'CN=Schema,CN=Configuration,${WIN_BASEDN}' ]:
 574         t.cmd_contains("bin/samba-tool drs replicate ${HOSTNAME}.${WIN_REALM} ${WIN_HOSTNAME}.${WIN_REALM} %s -k yes" % nc, ["was successful"])
 575         t.cmd_contains("bin/samba-tool drs replicate ${WIN_HOSTNAME}.${WIN_REALM} ${HOSTNAME}.${WIN_REALM} %s -k yes" % nc, ["was successful"])
 576
 577     retries = 10
 578     i = 1
 579     while i == 1 and retries > 0:
 580         child.sendline("net use t: \\\\${HOSTNAME}.${WIN_REALM}\\test")
 581         i = child.expect(["The command completed successfully", "The network path was not found"])
 582         child.expect("C:")
 583         if i == 1:
 584             time.sleep(2)
 585         retries -=1
 586
 587     t.info("Checking if showrepl is happy")
 588     child.sendline("repadmin /showrepl")
 589     child.expect("${WIN_BASEDN}")
 590     child.expect("was successful")
 591     child.expect("CN=Configuration,${WIN_BASEDN}")
 592     child.expect("was successful")
 593     child.expect("CN=Configuration,${WIN_BASEDN}")
 594     child.expect("was successful")
 595
 596     t.info("Checking if new users propogate to windows")
 597     t.retry_cmd('bin/samba-tool newuser test2 ${PASSWORD2}', ["created successfully"])
 598     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${WIN_REALM} -Utest2%${PASSWORD2} -k no", ['Sharename', 'Remote IPC'])
 599     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${WIN_REALM} -Utest2%${PASSWORD2} -k yes", ['Sharename', 'Remote IPC'])
 600
 601     t.info("Checking if new users on windows propogate to samba")
 602     child.sendline("net user test3 ${PASSWORD3} /add")
 603     child.expect("The command completed successfully")
 604     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k no", ['Sharename', 'IPC'])
 605     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k yes", ['Sharename', 'IPC'])
 606
 607     t.info("Checking propogation of user deletion")
 608     t.run_cmd('bin/samba-tool user delete test2 -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 609     child.sendline("net user test3 /del")
 610     child.expect("The command completed successfully")
 611
 612     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${WIN_REALM} -Utest2%${PASSWORD2} -k no", ['LOGON_FAILURE'])
 613     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k no", ['LOGON_FAILURE'])
 614     t.retry_cmd("bin/smbclient -L ${WIN_HOSTNAME}.${WIN_REALM} -Utest2%${PASSWORD2} -k yes", ['LOGON_FAILURE'])
 615     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k yes", ['LOGON_FAILURE'])
 616     t.vm_poweroff("${WIN_VM}")
 617
 618
 619 def join_as_rodc(t, vm):
 620     '''join a windows domain as a RODC'''
 621     t.setwinvars(vm)
 622     t.info("Joining ${WIN_VM} as a RODC using samba-tool join DC")
 623     t.port_wait("${WIN_IP}", 389)
 624     t.retry_cmd("host -t SRV _ldap._tcp.${WIN_REALM} ${WIN_IP}", ['has SRV record'] )
 625     t.retry_cmd("bin/samba-tool drs showrepl ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator%${WIN_PASS}", ['INBOUND NEIGHBORS'] )
 626     t.run_cmd('bin/samba-tool join ${WIN_REALM} RODC -Uadministrator%${WIN_PASS} -d${DEBUGLEVEL} --option=interfaces=${INTERFACE}')
 627     t.run_cmd('bin/samba-tool drs kcc ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 628
 629
 630 def test_join_as_rodc(t, vm):
 631     '''test a windows domain RODC join'''
 632     t.info("Checking the RODC join is OK")
 633     t.chdir('${PREFIX}')
 634     t.retry_cmd('bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}', ["C$", "IPC$", "Sharename"])
 635     t.cmd_contains("host -t A ${HOSTNAME}.${WIN_REALM}.", ['has address'])
 636     child = t.open_telnet("${WIN_HOSTNAME}", "${WIN_DOMAIN}\\administrator", "${WIN_PASS}", set_time=True)
 637
 638     t.info("Forcing kcc runs, and replication")
 639     t.run_cmd('bin/samba-tool drs kcc ${HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 640     t.run_cmd('bin/samba-tool drs kcc ${WIN_HOSTNAME}.${WIN_REALM} -Uadministrator@${WIN_REALM}%${WIN_PASS}')
 641
 642     t.kinit("administrator@${WIN_REALM}", "${WIN_PASS}")
 643     for nc in [ '${WIN_BASEDN}', 'CN=Configuration,${WIN_BASEDN}', 'CN=Schema,CN=Configuration,${WIN_BASEDN}' ]:
 644         t.cmd_contains("bin/samba-tool drs replicate ${HOSTNAME}.${WIN_REALM} ${WIN_HOSTNAME}.${WIN_REALM} %s -k yes" % nc, ["was successful"])
 645
 646     retries = 10
 647     i = 1
 648     while i == 1 and retries > 0:
 649         child.sendline("net use t: \\\\${HOSTNAME}.${WIN_REALM}\\test")
 650         i = child.expect(["The command completed successfully", "The network path was not found"])
 651         child.expect("C:")
 652         if i == 1:
 653             time.sleep(2)
 654         retries -=1
 655
 656     t.info("Checking if showrepl is happy")
 657     child.sendline("repadmin /showrepl")
 658     child.expect("DSA invocationID")
 659
 660     t.cmd_contains("bin/samba-tool drs showrepl ${WIN_HOSTNAME}.${WIN_REALM} -k yes",
 661                  [ "INBOUND NEIGHBORS",
 662                    "OUTBOUND NEIGHBORS",
 663                    "${WIN_BASEDN}",
 664                    "Last attempt .* was successful",
 665                    "CN=Configuration,${WIN_BASEDN}",
 666                    "Last attempt .* was successful",
 667                    "CN=Configuration,${WIN_BASEDN}",
 668                    "Last attempt .* was successful" ],
 669                    ordered=True,
 670                    regex=True)
 671
 672     t.info("Checking if new users on windows propogate to samba")
 673     child.sendline("net user test3 ${PASSWORD3} /add")
 674     child.expect("The command completed successfully")
 675     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k no", ['Sharename', 'IPC'])
 676     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k yes", ['Sharename', 'IPC'])
 677
 678     # should this work?
 679     t.info("Checking if new users propogate to windows")
 680     t.cmd_contains('bin/samba-tool newuser test2 ${PASSWORD2}', ['No RID Set DN'])
 681
 682     t.info("Checking propogation of user deletion")
 683     child.sendline("net user test3 /del")
 684     child.expect("The command completed successfully")
 685
 686     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k no", ['LOGON_FAILURE'])
 687     t.retry_cmd("bin/smbclient -L ${HOSTNAME}.${WIN_REALM} -Utest3%${PASSWORD3} -k yes", ['LOGON_FAILURE'])
 688     t.vm_poweroff("${WIN_VM}")
 689
 690
 691 def run_dcpromo_as_first_dc(t, vm, func_level=None):
 692     t.setwinvars(vm)
 693     t.info("Configuring a windows VM ${WIN_VM} at the first DC in the domain using dcpromo")
 694     child = t.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_time=True)
 695     child.sendline("dcdiag");
 696     if t.get_is_dc(child):
 697         return
 698
 699     if func_level == '2008r2':
 700         t.setvar("FUNCTION_LEVEL_INT", str(4))
 701     elif func_level == '2003':
 702         t.setvar("FUNCTION_LEVEL_INT", str(1))
 703     else:
 704         t.setvar("FUNCTION_LEVEL_INT", str(0))
 705
 706     child = t.open_telnet("${WIN_HOSTNAME}", "administrator", "${WIN_PASS}", set_ip=True)
 707     child.sendline("dcdiag");
 708
 709     """This server must therefore not yet be a directory server, so we must promote it"""
 710     child.sendline("copy /Y con answers.txt")
 711     child.sendline('''
 712 [DCInstall]
 713 ; New forest promotion
 714 ReplicaOrNewDomain=Domain
 715 NewDomain=Forest
 716 NewDomainDNSName=${WIN_REALM}
 717 ForestLevel=${FUNCTION_LEVEL_INT}
 718 DomainNetbiosName=${WIN_DOMAIN}
 719 DomainLevel=${FUNCTION_LEVEL_INT}
 720 InstallDNS=Yes
 721 ConfirmGc=Yes
 722 CreateDNSDelegation=No
 723 DatabasePath="C:\Windows\NTDS"
 724 LogPath="C:\Windows\NTDS"
 725 SYSVOLPath="C:\Windows\SYSVOL"
 726 ; Set SafeModeAdminPassword to the correct value prior to using the unattend file
 727 SafeModeAdminPassword=${WIN_PASS}
 728 ; Run-time flags (optional)
 729 RebootOnCompletion=No
 730 \1a
 731 ''')
 732     child.expect("copied.")
 733     child.expect("C:")
 734     child.expect("C:")
 735     child.sendline("dcpromo /answer:answers.txt")
 736     i = child.expect(["You must restart this computer", "failed", "Active Directory Domain Services was not installed", "C:"], timeout=120)
 737     if i == 1 or i == 2:
 738         raise Exception("dcpromo failed")
 739     child.sendline("shutdown -r -t 0")
 740     t.port_wait("${WIN_IP}", 139, wait_for_fail=True)
 741     t.port_wait("${WIN_IP}", 139)
 742
 743 def test_howto(t):
 744     '''test the Samba4 howto'''
 745
 746     check_prerequesites(t)
 747
 748     # we don't need fsync safety in these tests
 749     t.putenv('TDB_NO_FSYNC', '1')
 750
 751     if not t.skip("configure_bind"):
 752         configure_bind(t)
 753     if not t.skip("stop_bind"):
 754         stop_bind(t)
 755     if not t.skip("stop_vms"):
 756         stop_vms(t)
 757
 758     if not t.skip("build"):
 759         build_s4(t)
 760
 761     if not t.skip("provision"):
 762         provision_s4(t)
 763
 764     if not t.skip("create-shares"):
 765         create_shares(t)
 766
 767     if not t.skip("starts4"):
 768         start_s4(t)
 769     if not t.skip("smbclient"):
 770         test_smbclient(t)
 771     if not t.skip("configure_bind2"):
 772         configure_bind(t)
 773     if not t.skip("start_bind"):
 774         start_bind(t)
 775     if not t.skip("dns"):
 776         test_dns(t)
 777     if not t.skip("kerberos"):
 778         test_kerberos(t)
 779     if not t.skip("dyndns"):
 780         test_dyndns(t)
 781
 782     if t.have_vm('WINDOWS7') and not t.skip("windows7"):
 783         run_winjoin(t, "WINDOWS7")
 784         test_winjoin(t, "WINDOWS7")
 785
 786     if t.have_vm('WINXP') and not t.skip("winxp"):
 787         run_winjoin(t, "WINXP")
 788         test_winjoin(t, "WINXP")
 789
 790     if t.have_vm('W2K8R2C') and not t.skip("dcpromo_rodc"):
 791         t.info("Testing w2k8r2 RODC dcpromo")
 792         run_dcpromo_rodc(t, "W2K8R2C")
 793         test_dcpromo_rodc(t, "W2K8R2C")
 794
 795     if t.have_vm('W2K8R2B') and not t.skip("dcpromo_w2k8r2"):
 796         t.info("Testing w2k8r2 dcpromo")
 797         run_dcpromo(t, "W2K8R2B")
 798         test_dcpromo(t, "W2K8R2B")
 799
 800     if t.have_vm('W2K8B') and not t.skip("dcpromo_w2k8"):
 801         t.info("Testing w2k8 dcpromo")
 802         run_dcpromo(t, "W2K8B")
 803         test_dcpromo(t, "W2K8B")
 804
 805     if t.have_vm('W2K3B') and not t.skip("dcpromo_w2k3"):
 806         t.info("Testing w2k3 dcpromo")
 807         t.info("Changing to 2003 functional level")
 808         provision_s4(t, func_level='2003')
 809         create_shares(t)
 810         start_s4(t)
 811         test_smbclient(t)
 812         restart_bind(t)
 813         test_dns(t)
 814         test_kerberos(t)
 815         test_dyndns(t)
 816         run_dcpromo(t, "W2K3B")
 817         test_dcpromo(t, "W2K3B")
 818
 819     if t.have_vm('W2K8R2A') and not t.skip("join_w2k8r2"):
 820         prep_join_as_dc(t, "W2K8R2A")
 821         run_dcpromo_as_first_dc(t, "W2K8R2A", func_level='2008r2')
 822         join_as_dc(t, "W2K8R2A")
 823         create_shares(t)
 824         start_s4(t)
 825         test_dyndns(t)
 826         test_join_as_dc(t, "W2K8R2A")
 827
 828     if t.have_vm('W2K8R2A') and not t.skip("join_rodc"):
 829         prep_join_as_dc(t, "W2K8R2A")
 830         run_dcpromo_as_first_dc(t, "W2K8R2A", func_level='2008r2')
 831         join_as_rodc(t, "W2K8R2A")
 832         create_shares(t)
 833         start_s4(t)
 834         test_dyndns(t)
 835         test_join_as_rodc(t, "W2K8R2A")
 836
 837     if t.have_vm('W2K3A') and not t.skip("join_w2k3"):
 838         prep_join_as_dc(t, "W2K3A")
 839         run_dcpromo_as_first_dc(t, "W2K3A", func_level='2003')
 840         join_as_dc(t, "W2K3A")
 841         create_shares(t)
 842         start_s4(t)
 843         test_dyndns(t)
 844         test_join_as_dc(t, "W2K3A")
 845
 846     t.info("Howto test: All OK")
 847
 848
 849 def test_cleanup(t):
 850     '''cleanup after tests'''
 851     t.info("Cleaning up ...")
 852     restore_resolv_conf(t)
 853     if getattr(t, 'bind_child', False):
 854         t.bind_child.kill()
 855
 856
 857 if __name__ == '__main__':
 858     parser = optparse.OptionParser("test-howto.py")
 859     parser.add_option("--conf", type='string', default='', help='config file')
 860     parser.add_option("--skip", type='string', default='', help='list of steps to skip (comma separated)')
 861     parser.add_option("--vms", type='string', default=None, help='list of VMs to use (comma separated)')
 862     parser.add_option("--list", action='store_true', default=False, help='list the available steps')
 863     parser.add_option("--rebase", action='store_true', default=False, help='do a git pull --rebase')
 864     parser.add_option("--clean", action='store_true', default=False, help='clean the tree')
 865     parser.add_option("--prefix", type='string', default=None, help='override install prefix')
 866     parser.add_option("--sourcetree", type='string', default=None, help='override sourcetree location')
 867     parser.add_option("--nocleanup", action='store_true', default=False, help='disable cleanup code')
 868
 869     opts, args = parser.parse_args()
 870
 871     if not opts.conf:
 872         print("Please specify a config file with --conf")
 873         sys.exit(1)
 874
 875     t = wintest.wintest()
 876     t.load_config(opts.conf)
 877
 878     t.set_skip(opts.skip)
 879     t.set_vms(opts.vms)
 880
 881     if opts.list:
 882         t.list_steps_mode()
 883
 884     if opts.prefix:
 885         t.setvar('PREFIX', opts.prefix)
 886
 887     if opts.sourcetree:
 888         t.setvar('SOURCETREE', opts.sourcetree)
 889
 890     if opts.rebase:
 891         t.info('rebasing')
 892         t.chdir('${SOURCETREE}')
 893         t.run_cmd('git pull --rebase')
 894
 895     if opts.clean:
 896         t.info('rebasing')
 897         t.chdir('${SOURCETREE}/source4')
 898         t.run_cmd('rm -rf bin')
 899
 900     try:
 901         test_howto(t)
 902     except:
 903         if not opts.nocleanup:
 904             test_cleanup(t)
 905         raise
 906
 907     if not opts.nocleanup:
 908         test_cleanup(t)
 909     t.info("S4 howto test: All OK")