2 # subunit: extensions to Python unittest to get test results from subprocesses.
3 # Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
5 # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6 # license at the users choice. A copy of both licenses are available in the
7 # project source as Apache-2.0 and BSD. You may not use this file except in
8 # compliance with one of these two licences.
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # license you chose for the specific language governing permissions and
14 # limitations under that license.
19 from StringIO import StringIO
24 import subunit.iso8601 as iso8601
25 import subunit.test_results
26 from subunit.content_type import ContentType
27 from subunit.content import Content
30 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
32 def __init__(self, decorated):
34 super(LoggingDecorator, self).__init__(decorated)
36 def _before_event(self):
40 class AssertBeforeTestResult(LoggingDecorator):
41 """A TestResult for checking preconditions."""
43 def __init__(self, decorated, test):
45 super(AssertBeforeTestResult, self).__init__(decorated)
47 def _before_event(self):
48 self.test.assertEqual(1, self.earlier._calls)
49 super(AssertBeforeTestResult, self)._before_event()
52 class TimeCapturingResult(unittest.TestResult):
55 super(TimeCapturingResult, self).__init__()
58 def time(self, a_datetime):
59 self._calls.append(a_datetime)
62 class LoggingResult(object):
63 """Basic support for logging of results."""
69 class Python26TestResult(LoggingResult):
70 """A python 2.6 like test result, that logs."""
72 def addError(self, test, err):
73 self._calls.append(('addError', test, err))
75 def addFailure(self, test, err):
76 self._calls.append(('addFailure', test, err))
78 def addSuccess(self, test):
79 self._calls.append(('addSuccess', test))
81 def startTest(self, test):
82 self._calls.append(('startTest', test))
84 def stopTest(self, test):
85 self._calls.append(('stopTest', test))
88 class Python27TestResult(Python26TestResult):
89 """A python 2.7 like test result, that logs."""
91 def addExpectedFailure(self, test, err):
92 self._calls.append(('addExpectedFailure', test, err))
94 def addSkip(self, test, reason):
95 self._calls.append(('addSkip', test, reason))
97 def addUnexpectedSuccess(self, test):
98 self._calls.append(('addUnexpectedSuccess', test))
100 def startTestRun(self):
101 self._calls.append(('startTestRun',))
103 def stopTestRun(self):
104 self._calls.append(('stopTestRun',))
107 class ExtendedTestResult(Python27TestResult):
108 """A test result like the proposed extended unittest result API."""
110 def addError(self, test, err=None, details=None):
111 self._calls.append(('addError', test, err or details))
113 def addFailure(self, test, err=None, details=None):
114 self._calls.append(('addFailure', test, err or details))
116 def addExpectedFailure(self, test, err=None, details=None):
117 self._calls.append(('addExpectedFailure', test, err or details))
119 def addSkip(self, test, reason=None, details=None):
120 self._calls.append(('addSkip', test, reason or details))
122 def addSuccess(self, test, details=None):
124 self._calls.append(('addSuccess', test, details))
126 self._calls.append(('addSuccess', test))
128 def addUnexpectedSuccess(self, test, details=None):
130 self._calls.append(('addUnexpectedSuccess', test, details))
132 self._calls.append(('addUnexpectedSuccess', test))
135 class TestExtendedToOriginalResultDecoratorBase(unittest.TestCase):
137 def make_26_result(self):
138 self.result = Python26TestResult()
139 self.make_converter()
141 def make_27_result(self):
142 self.result = Python27TestResult()
143 self.make_converter()
145 def make_converter(self):
147 subunit.test_results.ExtendedToOriginalDecorator(self.result)
149 def make_extended_result(self):
150 self.result = ExtendedTestResult()
151 self.make_converter()
153 def check_outcome_details(self, outcome):
154 """Call an outcome with a details dict to be passed through."""
155 # This dict is /not/ convertible - thats deliberate, as it should
156 # not hit the conversion code path.
157 details = {'foo': 'bar'}
158 getattr(self.converter, outcome)(self, details=details)
159 self.assertEqual([(outcome, self, details)], self.result._calls)
161 def get_details_and_string(self):
162 """Get a details dict and expected string."""
163 text1 = lambda:["1\n2\n"]
164 text2 = lambda:["3\n4\n"]
165 bin1 = lambda:["5\n"]
166 details = {'text 1': Content(ContentType('text', 'plain'), text1),
167 'text 2': Content(ContentType('text', 'strange'), text2),
168 'bin 1': Content(ContentType('application', 'binary'), bin1)}
169 return (details, "Binary content: bin 1\n"
170 "Text attachment: text 1\n------------\n1\n2\n"
171 "------------\nText attachment: text 2\n------------\n"
172 "3\n4\n------------\n")
174 def check_outcome_details_to_exec_info(self, outcome, expected=None):
175 """Call an outcome with a details dict to be made into exc_info."""
176 # The conversion is a done using RemoteError and the string contents
177 # of the text types in the details dict.
180 details, err_str = self.get_details_and_string()
181 getattr(self.converter, outcome)(self, details=details)
182 err = subunit.RemoteError(err_str)
183 self.assertEqual([(expected, self, err)], self.result._calls)
185 def check_outcome_details_to_nothing(self, outcome, expected=None):
186 """Call an outcome with a details dict to be swallowed."""
189 details = {'foo': 'bar'}
190 getattr(self.converter, outcome)(self, details=details)
191 self.assertEqual([(expected, self)], self.result._calls)
193 def check_outcome_details_to_string(self, outcome):
194 """Call an outcome with a details dict to be stringified."""
195 details, err_str = self.get_details_and_string()
196 getattr(self.converter, outcome)(self, details=details)
197 self.assertEqual([(outcome, self, err_str)], self.result._calls)
199 def check_outcome_exc_info(self, outcome, expected=None):
200 """Check that calling a legacy outcome still works."""
201 # calling some outcome with the legacy exc_info style api (no keyword
202 # parameters) gets passed through.
205 err = subunit.RemoteError("foo\nbar\n")
206 getattr(self.converter, outcome)(self, err)
207 self.assertEqual([(expected, self, err)], self.result._calls)
209 def check_outcome_nothing(self, outcome, expected=None):
210 """Check that calling a legacy outcome still works."""
213 getattr(self.converter, outcome)(self)
214 self.assertEqual([(expected, self)], self.result._calls)
216 def check_outcome_string_nothing(self, outcome, expected):
217 """Check that calling outcome with a string calls expected."""
218 getattr(self.converter, outcome)(self, "foo")
219 self.assertEqual([(expected, self)], self.result._calls)
221 def check_outcome_string(self, outcome):
222 """Check that calling outcome with a string works."""
223 getattr(self.converter, outcome)(self, "foo")
224 self.assertEqual([(outcome, self, "foo")], self.result._calls)
227 class TestExtendedToOriginalResultDecorator(
228 TestExtendedToOriginalResultDecoratorBase):
230 def test_startTest_py26(self):
231 self.make_26_result()
232 self.converter.startTest(self)
233 self.assertEqual([('startTest', self)], self.result._calls)
235 def test_startTest_py27(self):
236 self.make_27_result()
237 self.converter.startTest(self)
238 self.assertEqual([('startTest', self)], self.result._calls)
240 def test_startTest_pyextended(self):
241 self.make_extended_result()
242 self.converter.startTest(self)
243 self.assertEqual([('startTest', self)], self.result._calls)
245 def test_startTestRun_py26(self):
246 self.make_26_result()
247 self.converter.startTestRun()
248 self.assertEqual([], self.result._calls)
250 def test_startTestRun_py27(self):
251 self.make_27_result()
252 self.converter.startTestRun()
253 self.assertEqual([('startTestRun',)], self.result._calls)
255 def test_startTestRun_pyextended(self):
256 self.make_extended_result()
257 self.converter.startTestRun()
258 self.assertEqual([('startTestRun',)], self.result._calls)
260 def test_stopTest_py26(self):
261 self.make_26_result()
262 self.converter.stopTest(self)
263 self.assertEqual([('stopTest', self)], self.result._calls)
265 def test_stopTest_py27(self):
266 self.make_27_result()
267 self.converter.stopTest(self)
268 self.assertEqual([('stopTest', self)], self.result._calls)
270 def test_stopTest_pyextended(self):
271 self.make_extended_result()
272 self.converter.stopTest(self)
273 self.assertEqual([('stopTest', self)], self.result._calls)
275 def test_stopTestRun_py26(self):
276 self.make_26_result()
277 self.converter.stopTestRun()
278 self.assertEqual([], self.result._calls)
280 def test_stopTestRun_py27(self):
281 self.make_27_result()
282 self.converter.stopTestRun()
283 self.assertEqual([('stopTestRun',)], self.result._calls)
285 def test_stopTestRun_pyextended(self):
286 self.make_extended_result()
287 self.converter.stopTestRun()
288 self.assertEqual([('stopTestRun',)], self.result._calls)
291 class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase):
295 def test_outcome_Original_py26(self):
296 self.make_26_result()
297 self.check_outcome_exc_info(self.outcome)
299 def test_outcome_Original_py27(self):
300 self.make_27_result()
301 self.check_outcome_exc_info(self.outcome)
303 def test_outcome_Original_pyextended(self):
304 self.make_extended_result()
305 self.check_outcome_exc_info(self.outcome)
307 def test_outcome_Extended_py26(self):
308 self.make_26_result()
309 self.check_outcome_details_to_exec_info(self.outcome)
311 def test_outcome_Extended_py27(self):
312 self.make_27_result()
313 self.check_outcome_details_to_exec_info(self.outcome)
315 def test_outcome_Extended_pyextended(self):
316 self.make_extended_result()
317 self.check_outcome_details(self.outcome)
319 def test_outcome__no_details(self):
320 self.make_extended_result()
321 self.assertRaises(ValueError,
322 getattr(self.converter, self.outcome), self)
325 class TestExtendedToOriginalAddFailure(
326 TestExtendedToOriginalAddError):
328 outcome = 'addFailure'
331 class TestExtendedToOriginalAddExpectedFailure(
332 TestExtendedToOriginalAddError):
334 outcome = 'addExpectedFailure'
336 def test_outcome_Original_py26(self):
337 self.make_26_result()
338 self.check_outcome_exc_info(self.outcome, 'addFailure')
340 def test_outcome_Extended_py26(self):
341 self.make_26_result()
342 self.check_outcome_details_to_exec_info(self.outcome, 'addFailure')
346 class TestExtendedToOriginalAddSkip(
347 TestExtendedToOriginalResultDecoratorBase):
351 def test_outcome_Original_py26(self):
352 self.make_26_result()
353 self.check_outcome_string_nothing(self.outcome, 'addSuccess')
355 def test_outcome_Original_py27(self):
356 self.make_27_result()
357 self.check_outcome_string(self.outcome)
359 def test_outcome_Original_pyextended(self):
360 self.make_extended_result()
361 self.check_outcome_string(self.outcome)
363 def test_outcome_Extended_py26(self):
364 self.make_26_result()
365 self.check_outcome_string_nothing(self.outcome, 'addSuccess')
367 def test_outcome_Extended_py27(self):
368 self.make_27_result()
369 self.check_outcome_details_to_string(self.outcome)
371 def test_outcome_Extended_pyextended(self):
372 self.make_extended_result()
373 self.check_outcome_details(self.outcome)
375 def test_outcome__no_details(self):
376 self.make_extended_result()
377 self.assertRaises(ValueError,
378 getattr(self.converter, self.outcome), self)
381 class TestExtendedToOriginalAddSuccess(
382 TestExtendedToOriginalResultDecoratorBase):
384 outcome = 'addSuccess'
385 expected = 'addSuccess'
387 def test_outcome_Original_py26(self):
388 self.make_26_result()
389 self.check_outcome_nothing(self.outcome, self.expected)
391 def test_outcome_Original_py27(self):
392 self.make_27_result()
393 self.check_outcome_nothing(self.outcome)
395 def test_outcome_Original_pyextended(self):
396 self.make_extended_result()
397 self.check_outcome_nothing(self.outcome)
399 def test_outcome_Extended_py26(self):
400 self.make_26_result()
401 self.check_outcome_details_to_nothing(self.outcome, self.expected)
403 def test_outcome_Extended_py27(self):
404 self.make_27_result()
405 self.check_outcome_details_to_nothing(self.outcome)
407 def test_outcome_Extended_pyextended(self):
408 self.make_extended_result()
409 self.check_outcome_details(self.outcome)
412 class TestExtendedToOriginalAddUnexpectedSuccess(
413 TestExtendedToOriginalAddSuccess):
415 outcome = 'addUnexpectedSuccess'
418 class TestHookedTestResultDecorator(unittest.TestCase):
421 # And end to the chain
422 terminal = unittest.TestResult()
423 # Asserts that the call was made to self.result before asserter was
425 asserter = AssertBeforeTestResult(terminal, self)
426 # The result object we call, which much increase its call count.
427 self.result = LoggingDecorator(asserter)
428 asserter.earlier = self.result
431 # The hook in self.result must have been called
432 self.assertEqual(1, self.result._calls)
433 # The hook in asserter must have been called too, otherwise the
434 # assertion about ordering won't have completed.
435 self.assertEqual(1, self.result.decorated._calls)
437 def test_startTest(self):
438 self.result.startTest(self)
440 def test_startTestRun(self):
441 self.result.startTestRun()
443 def test_stopTest(self):
444 self.result.stopTest(self)
446 def test_stopTestRun(self):
447 self.result.stopTestRun()
449 def test_addError(self):
450 self.result.addError(self, subunit.RemoteError())
452 def test_addFailure(self):
453 self.result.addFailure(self, subunit.RemoteError())
455 def test_addSuccess(self):
456 self.result.addSuccess(self)
458 def test_addSkip(self):
459 self.result.addSkip(self, "foo")
461 def test_addExpectedFailure(self):
462 self.result.addExpectedFailure(self, subunit.RemoteError())
464 def test_addUnexpectedSuccess(self):
465 self.result.addUnexpectedSuccess(self)
467 def test_progress(self):
468 self.result.progress(1, subunit.PROGRESS_SET)
470 def test_wasSuccessful(self):
471 self.result.wasSuccessful()
473 def test_shouldStop(self):
474 self.result.shouldStop
480 self.result.time(None)
483 class TestAutoTimingTestResultDecorator(unittest.TestCase):
486 # And end to the chain which captures time events.
487 terminal = TimeCapturingResult()
488 # The result object under test.
489 self.result = subunit.test_results.AutoTimingTestResultDecorator(
492 def test_without_time_calls_time_is_called_and_not_None(self):
493 self.result.startTest(self)
494 self.assertEqual(1, len(self.result.decorated._calls))
495 self.assertNotEqual(None, self.result.decorated._calls[0])
497 def test_no_time_from_progress(self):
498 self.result.progress(1, subunit.PROGRESS_CUR)
499 self.assertEqual(0, len(self.result.decorated._calls))
501 def test_no_time_from_shouldStop(self):
502 self.result.decorated.stop()
503 self.result.shouldStop
504 self.assertEqual(0, len(self.result.decorated._calls))
506 def test_calling_time_inhibits_automatic_time(self):
507 # Calling time() outputs a time signal immediately and prevents
508 # automatically adding one when other methods are called.
509 time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
510 self.result.time(time)
511 self.result.startTest(self)
512 self.result.stopTest(self)
513 self.assertEqual(1, len(self.result.decorated._calls))
514 self.assertEqual(time, self.result.decorated._calls[0])
516 def test_calling_time_None_enables_automatic_time(self):
517 time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
518 self.result.time(time)
519 self.assertEqual(1, len(self.result.decorated._calls))
520 self.assertEqual(time, self.result.decorated._calls[0])
521 # Calling None passes the None through, in case other results care.
522 self.result.time(None)
523 self.assertEqual(2, len(self.result.decorated._calls))
524 self.assertEqual(None, self.result.decorated._calls[1])
525 # Calling other methods doesn't generate an automatic time event.
526 self.result.startTest(self)
527 self.assertEqual(3, len(self.result.decorated._calls))
528 self.assertNotEqual(None, self.result.decorated._calls[2])
532 loader = subunit.tests.TestUtil.TestLoader()
533 result = loader.loadTestsFromName(__name__)