selftest: Have only one set of selftest knownfail and skip files

Listing tests that are not ever proposed (eg samba4.* in the samba3 selftest)
is not an error, so just combine the lists.

This is being done because some folks trying to learn how our 'make
test' works are having trouble following the distributed nature of the
selftest system.

Andrew Bartlett

diff --git a/source4/selftest/knownfail b/selftest/knownfail
similarity index 78%
rename from source4/selftest/knownfail
rename to selftest/knownfail
index 384ed1b90bc33c177b9c9b6b8dfa005c34f2f067..7f023211f6e13fdfd936f95c272101082f151008 100644 (file)
--- a/source4/selftest/knownfail
+++ b/selftest/knownfail
@@ -1,8 +1,24 @@
-# This file contains a list of regular expressions matching the names of 
-# tests that are expected to fail. 
+# This file contains a list of regular expressions matching the names of
+# tests that are expected to fail.
 #
 # "make test" will not report failures for tests listed here and will consider
 # a successful run for any of these tests an error.
+
+^samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
+.*printer.*print_test_extended # fails on some hosts due to timing issues ?
+.*printer.*print_test # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
+^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
+^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
+^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
+.*driver.add_driver_timestamps # we only can store dates, not timestamps
+^samba3.raw.mux.* #This test is flaky on the async lock time
+^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
+^samba3.posix_s3.nbt.dgram.*netlogon2
+^samba3.*rap.sam.*.useradd # Not provided by Samba 3
+^samba3.*rap.sam.*.userdelete # Not provided by Samba 3
+^samba3.*rap.basic.*.netsessiongetinfo # Not provided by Samba 3
+^samba3.posix_s3.libsmbclient .opendir # This requires a workgroup called 'WORKGROUP' and for netbios browse lists to have been registered
 ^samba4.local.resolve.*.async
 ^samba4.local.iconv.*.next_codepoint()
 # these show that we still have some differences between our system

diff --git a/source4/selftest/quick b/selftest/quick
similarity index 96%
rename from source4/selftest/quick
rename to selftest/quick
index 20970857a490c7251f0f04e4ed83ef006e82febb..7605f3f88774eb21b251922bb92c39e489695169 100644 (file)
--- a/source4/selftest/quick
+++ b/selftest/quick
@@ -1,5 +1,5 @@
-# This file contains regexes matching the tests that should be run 
-# when doing a "quicktest" - verifying whether the build is working 
+# This file contains regexes matching the tests that should be run
+# when doing a "quicktest" - verifying whether the build is working
 # rather than trying to see what exactly is broken.
 #
 # This should be as quick as possible but cover as much code as possible.
@@ -20,7 +20,7 @@ raw.close
 raw.ioctl
 raw.rename
 raw.eas
-base.open 
+base.open
 rpc.altercontext
 rpc.join
 rpc.echo

diff --git a/source4/selftest/skip b/selftest/skip
similarity index 84%
rename from source4/selftest/skip
rename to selftest/skip
index e19eae74a9bd741b8756eb53e098860841b89ae1..686ccdb0e731b45aa137afd1e5e4d3061e21c1fe 100644 (file)
--- a/source4/selftest/skip
+++ b/selftest/skip
@@ -1,4 +1,4 @@
-# This file contains a list of regular expressions matching testsuites that 
+# This file contains a list of regular expressions matching testsuites that
 # should be skipped during "make test".
 #
 # Possible reasons for adding a testsuite here:
@@ -8,7 +8,7 @@
 # * Testsuite contains "flapping" tests (sometimes success, sometimes failure)
 # * Testsuite hangs indefinitely
 #
-# If a testsuite is partially succeeding, please list the failing bits 
+# If a testsuite is partially succeeding, please list the failing bits
 # in the selftest/knownfail file rather than disabling the testsuite completely.
 # That way those tests that do succeed still get run and we will be notified
 # if a known failing test suddenly starts succeeding.
@@ -18,6 +18,28 @@
 #
 # Please add a comment for each testsuite you disable explaining why
 # it is being skipped.
+^samba3.smbtorture_s3.*.randomipc
+^samba3.smbtorture_s3.*.negnowait
+^samba3.smbtorture_s3.*.nbench
+^samba3.smbtorture_s3.*.errmapextract
+^samba3.smbtorture_s3.*.trans2scan
+^samba3.smbtorture_s3.*.nttransscan
+^samba3.smbtorture_s3.*.deny1
+^samba3.smbtorture_s3.*.deny2
+^samba3.smbtorture_s3.*.openattr
+^samba3.smbtorture_s3.*.casetable
+^samba3.smbtorture_s3.*.eatest
+^samba3.smbtorture_s3.*.mangle
+^samba3.smbtorture_s3.*.utable
+^samba3.smbtorture_s3.*.pipe_number
+^samba3.*base.charset
+^samba3.*raw.acls
+^samba3.*raw.composite
+^samba3.*raw.context
+^samba3.*raw.ioctl
+^samba3.*raw.qfileinfo
+^samba3.*raw.qfsinfo
+^samba3.*raw.sfileinfo.base
 ^samba4.raw.composite
 ^samba4.base.iometer
 ^samba4.base.casetable

diff --git a/source4/selftest/slow b/selftest/slow
similarity index 100%
rename from source4/selftest/slow
rename to selftest/slow

diff --git a/selftest/wscript b/selftest/wscript
index c34658a1a7b73742f6f4c18a2f44e49f709717a8..4d3d8f2841ff1fade633c4765a80326084492737 100644 (file)
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -70,19 +70,6 @@ def set_options(opt):
 def configure(conf):
     conf.env.SELFTEST_PREFIX = Options.options.SELFTEST_PREFIX
 
-def combine_files(file1, file2, outfile):
-
-    f1 = open(file1)
-    f2 = open(file2)
-    of = open(outfile, mode='w')
-    for line in f1:
-        of.write(line)
-    for line in f2:
-        of.write(line)
-    f1.close()
-    f2.close()
-    of.close()
-
 def cmd_testonly(opt):
     '''run tests without doing a build first'''
     env = LOAD_ENVIRONMENT()
@@ -101,7 +88,7 @@ def cmd_testonly(opt):
     env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
     if not env.SUBUNIT_FORMATTER:
         env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
-    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${SELFTEST_DIR}/knownfail'
+    env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail'
 
     if Options.options.FAIL_IMMEDIATELY:
         env.FILTER_XFAIL += ' --fail-immediately'
@@ -123,9 +110,9 @@ def cmd_testonly(opt):
 
     env.OPTIONS = '--binary-mapping=%s' % binary_mapping 
     if not Options.options.SLOWTEST:
-        env.OPTIONS += ' --exclude=${SELFTEST_DIR}/slow'
+        env.OPTIONS += ' --exclude=${srcdir}/selftest/slow'
     if Options.options.QUICKTEST:
-        env.OPTIONS += ' --quick --include=${SELFTEST_DIR}/quick'
+        env.OPTIONS += ' --quick --include=${srcdir}/selftest/quick'
     if Options.options.LOAD_LIST:
         env.OPTIONS += ' --load-list=%s' % Options.options.LOAD_LIST
     if Options.options.TESTENV:
@@ -185,20 +172,11 @@ def cmd_testonly(opt):
     if not os.path.isdir(env.SELFTEST_PREFIX):
         os.makedirs(env.SELFTEST_PREFIX, int('755', 8))
 
-    env.SELFTEST_TARGET = "samba"
-
-    for f in ["knownfail", "slow", "quick", "skip" ]:
-        combine_files(env.srcdir + "/source4/selftest/" + f,
-                      env.srcdir + "/source3/selftest/" + f,
-                      env.SELFTEST_PREFIX + "/" + f)
-
-    env.SELFTEST_DIR = env.SELFTEST_PREFIX
-
     env.TESTLISTS = ('--testlist="${PYTHON} ${srcdir}/source3/selftest/tests.py|" ' +
                      '--testlist="${PYTHON} ${srcdir}/source4/selftest/tests.py|"')
 
     # We use the full path rather than relative path because it cause problems on some plateforms (ie. solaris 8).
-    cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=samba --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${SELFTEST_DIR}/skip ${TESTLISTS} ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
+    cmd = '(${PERL} ${srcdir}/selftest/selftest.pl --target=samba --prefix=${SELFTEST_PREFIX} --srcdir=${srcdir} --exclude=${srcdir}/selftest/skip ${TESTLISTS} ${OPTIONS} --socket-wrapper ${TESTS} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS} | tee ${SELFTEST_PREFIX}/subunit'
 
     if os.environ.get('RUN_FROM_BUILD_FARM') is None and not Options.options.FILTERED_SUBUNIT:
         cmd += ' | ${FORMAT_TEST_OUTPUT}'

diff --git a/source3/selftest/knownfail b/source3/selftest/knownfail
deleted file mode 100644 (file)
index 4f746c8..0000000
--- a/source3/selftest/knownfail
+++ /dev/null
@@ -1,16 +0,0 @@
-^samba3.blackbox.failure # this is designed to fail, for testing our test infrastructure
-.*printer.*print_test_extended # fails on some hosts due to timing issues ?
-.*printer.*print_test # fails on some hosts due to timing issues ?
-^samba3.posix_s3.rap.printing # fails sometimes on sn-devel
-^samba3.posix_s3.rpc.spoolss.*printserver.enum_printers_old # fails on some hosts due to timing issues ?
-^samba3.posix_s3.rpc.spoolss.printer.*addprinterex.print_test # another intermittent failure
-^samba3.posix_s3.smb2.lock.*.rw-exclusive # another intermittent failure
-.*driver.add_driver_timestamps # we only can store dates, not timestamps
-^samba3.raw.mux.* #This test is flaky on the async lock time
-^samba3.smbtorture_s3.*OPLOCK4 # fails sometimes on sn-devel
-^samba3.posix_s3.nbt.dgram.*netlogon2
-^samba3.*rap.sam.*.useradd # Not provided by Samba 3
-^samba3.*rap.sam.*.userdelete # Not provided by Samba 3
-^samba3.*rap.basic.*.netsessiongetinfo # Not provided by Samba 3
-^samba3.posix_s3.libsmbclient .opendir # This requires a workgroup called 'WORKGROUP' and for netbios browse lists to have been registered
-

diff --git a/source3/selftest/quick b/source3/selftest/quick
deleted file mode 100644 (file)
index bb3b066..0000000
--- a/source3/selftest/quick
+++ /dev/null
@@ -1 +0,0 @@
-#none

diff --git a/source3/selftest/s3-selftest.sh b/source3/selftest/s3-selftest.sh
index a9ff875da2bbe9803783c388f5d5d6a6144c4d30..4943b1e2a62403710bcce0fdcac776cc3e22764a 100755 (executable)
--- a/source3/selftest/s3-selftest.sh
+++ b/source3/selftest/s3-selftest.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SOURCEDIR}/selftest/knownfail"
+FILTER_XFAIL="${PYTHON} -u ${SELFTESTDIR}/filter-subunit --expected-failures=${SELFTESTDIR}/knownfail"
 if [ "x${SUBUNIT_FORMATTER}" = x"" ]; then
        SUBUNIT_FORMATTER="${PYTHON} -u ${SELFTESTDIR}/format-subunit --prefix=${SELFTESTPREFIX} --immediate"
 fi
@@ -23,7 +23,7 @@ if [ "x${RUN_FROM_BUILD_FARM}" = "xyes" ]; then
                        --binary-mapping=smbtorture3:smbtorture,nmblookup3:nmblookup,smbclient3:smbclient,ntlm_auth3:ntlm_auth \
                        --prefix=${SELFTESTPREFIX} --target=samba3 \
                        --testlist="${PYTHON} ${SOURCEDIR}/selftest/tests.py|" \
-                       --exclude=${SOURCEDIR}/selftest/skip \
+                       --exclude=${SELFTESTDIR}/skip \
                        --srcdir="${SOURCEDIR}/.." \
                        --socket-wrapper ${TESTS} \
        && touch ${SELFTESTPREFIX}/st_done ) | \
@@ -37,7 +37,7 @@ else
                        --binary-mapping=smbtorture3:smbtorture,nmblookup3:nmblookup,smbclient3:smbclient,ntlm_auth3:ntlm_auth \
                        --prefix=${SELFTESTPREFIX} --target=samba3 \
                        --testlist="${PYTHON} ${SOURCEDIR}/selftest/tests.py|" \
-                       --exclude=${SOURCEDIR}/selftest/skip \
+                       --exclude=${SELFTESTDIR}/skip \
                        --srcdir="${SOURCEDIR}/.." \
                        --socket-wrapper ${TESTS} \
        && touch ${SELFTESTPREFIX}/st_done ) | \

diff --git a/source3/selftest/skip b/source3/selftest/skip
deleted file mode 100644 (file)
index 5e68172..0000000
--- a/source3/selftest/skip
+++ /dev/null
@@ -1,22 +0,0 @@
-^samba3.smbtorture_s3.*.randomipc
-^samba3.smbtorture_s3.*.negnowait
-^samba3.smbtorture_s3.*.nbench
-^samba3.smbtorture_s3.*.errmapextract
-^samba3.smbtorture_s3.*.trans2scan
-^samba3.smbtorture_s3.*.nttransscan
-^samba3.smbtorture_s3.*.deny1
-^samba3.smbtorture_s3.*.deny2
-^samba3.smbtorture_s3.*.openattr
-^samba3.smbtorture_s3.*.casetable
-^samba3.smbtorture_s3.*.eatest
-^samba3.smbtorture_s3.*.mangle
-^samba3.smbtorture_s3.*.utable
-^samba3.smbtorture_s3.*.pipe_number
-^samba3.*base.charset
-^samba3.*raw.acls
-^samba3.*raw.composite
-^samba3.*raw.context
-^samba3.*raw.ioctl
-^samba3.*raw.qfileinfo
-^samba3.*raw.qfsinfo
-^samba3.*raw.sfileinfo.base

diff --git a/source3/selftest/slow b/source3/selftest/slow
deleted file mode 100644 (file)
index bb3b066..0000000
--- a/source3/selftest/slow
+++ /dev/null
@@ -1 +0,0 @@
-#none