parser.add_option("--expected-failures", type="string",
help="File containing list of regexes matching tests to consider known "
"failures")
-parser.add_option("--strip-passed-output", action="store_true",
+parser.add_option("--flapping", type="string",
+ help="File containing list of flapping tests, of which to ignore results.")
+parser.add_option("--strip-passed-output", action="store_true",
help="Whether to strip output from tests that passed")
-parser.add_option("--fail-immediately", action="store_true",
+parser.add_option("--fail-immediately", action="store_true",
help="Whether to stop on the first error", default=False)
parser.add_option("--prefix", type="string",
help="Add prefix to all test names")
else:
expected_failures = {}
+
+if opts.flapping:
+ flapping = subunithelper.read_test_regexes(opts.flapping)
+else:
+ flapping = []
+
statistics = {
'TESTS_UNEXPECTED_OK': 0,
'TESTS_EXPECTED_OK': 0,
out = subunithelper.SubunitOps(sys.stdout)
msg_ops = subunithelper.FilterOps(out, opts.prefix, expected_failures,
opts.strip_passed_output,
- fail_immediately=opts.fail_immediately)
+ fail_immediately=opts.fail_immediately,
+ flapping=flapping)
try:
ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
def addFailure(self, test, details=None):
test = self._add_prefix(test)
xfail_reason = find_in_list(self.expected_failures, test.id())
+ if xfail_reason is None:
+ xfail_reason = find_in_list(self.flapping, test.id())
if xfail_reason is not None:
self.xfail_added+=1
self.total_xfail+=1
def addSuccess(self, test, details=None):
test = self._add_prefix(test)
- self._ops.addSuccess(test, details)
+ xfail_reason = find_in_list(self.expected_failures, test.id())
+ if xfail_reason is not None:
+ self.uxsuccess_added += 1
+ self.total_uxsuccess += 1
+ if details is not None:
+ details = subunit.RemoteError(unicode(details[1]) + xfail_reason.decode("utf-8"))
+ else:
+ details = subunit.RemoteError(xfail_reason.decode("utf-8"))
+ self._ops.addUnexpectedSuccess(test, details)
+ if self.output:
+ self._ops.output_msg(self.output)
+ if self.fail_immediately:
+ raise ImmediateFail()
+ else:
+ self._ops.addSuccess(test, details)
self.output = None
def skip_testsuite(self, name, reason=None):
self.error_added = 0
self.fail_added = 0
self.xfail_added = 0
+ self.uxsuccess_added = 0
def end_testsuite(self, name, result, reason=None):
xfail = False
self._ops.end_testsuite(name, result, reason)
def __init__(self, out, prefix=None, expected_failures=None,
- strip_ok_output=False, fail_immediately=False):
+ strip_ok_output=False, fail_immediately=False,
+ flapping=None):
self._ops = out
self.seen_output = False
self.output = None
self.expected_failures = expected_failures
else:
self.expected_failures = {}
+ if flapping is not None:
+ self.flapping = flapping
+ else:
+ self.flapping = {}
self.strip_ok_output = strip_ok_output
self.xfail_added = 0
self.fail_added = 0
+ self.uxsuccess_added = 0
self.total_xfail = 0
self.total_error = 0
self.total_fail = 0
+ self.total_uxsuccess = 0
self.error_added = 0
self.fail_immediately = fail_immediately
env.SUBUNIT_FORMATTER = os.getenv('SUBUNIT_FORMATTER')
if not env.SUBUNIT_FORMATTER:
env.SUBUNIT_FORMATTER = '${PYTHON} -u ${srcdir}/selftest/format-subunit --prefix=${SELFTEST_PREFIX} --immediate'
- env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --expected-failure=${srcdir}/selftest/flapping'
+ env.FILTER_XFAIL = '${PYTHON} -u ${srcdir}/selftest/filter-subunit --expected-failures=${srcdir}/selftest/knownfail --flapping=${srcdir}/selftest/flapping'
if Options.options.FAIL_IMMEDIATELY:
env.FILTER_XFAIL += ' --fail-immediately'