8ffdd4266cd9ec43909cbc249fd106453903ba6b
[third_party/subunit] / python / subunit / tests / test_test_results.py
1 #
2 #  subunit: extensions to Python unittest to get test results from subprocesses.
3 #  Copyright (C) 2009  Robert Collins <robertc@robertcollins.net>
4 #
5 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6 #  license at the users choice. A copy of both licenses are available in the
7 #  project source as Apache-2.0 and BSD. You may not use this file except in
8 #  compliance with one of these two licences.
9 #
10 #  Unless required by applicable law or agreed to in writing, software
11 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
13 #  license you chose for the specific language governing permissions and
14 #  limitations under that license.
15 #
16
17 import csv
18 import datetime
19 import sys
20 import unittest
21
22 from testtools import TestCase
23 from testtools.compat import StringIO
24 from testtools.content import (
25     text_content,
26     TracebackContent,
27     )
28 from testtools.testresult.doubles import ExtendedTestResult
29
30 import subunit
31 import subunit.iso8601 as iso8601
32 import subunit.test_results
33
34 import testtools
35
36
37 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
38
39     def __init__(self, decorated):
40         self._calls = 0
41         super(LoggingDecorator, self).__init__(decorated)
42
43     def _before_event(self):
44         self._calls += 1
45
46
47 class AssertBeforeTestResult(LoggingDecorator):
48     """A TestResult for checking preconditions."""
49
50     def __init__(self, decorated, test):
51         self.test = test
52         super(AssertBeforeTestResult, self).__init__(decorated)
53
54     def _before_event(self):
55         self.test.assertEqual(1, self.earlier._calls)
56         super(AssertBeforeTestResult, self)._before_event()
57
58
59 class TimeCapturingResult(unittest.TestResult):
60
61     def __init__(self):
62         super(TimeCapturingResult, self).__init__()
63         self._calls = []
64
65     def time(self, a_datetime):
66         self._calls.append(a_datetime)
67
68
69 class TestHookedTestResultDecorator(unittest.TestCase):
70
71     def setUp(self):
72         # An end to the chain
73         terminal = unittest.TestResult()
74         # Asserts that the call was made to self.result before asserter was
75         # called.
76         asserter = AssertBeforeTestResult(terminal, self)
77         # The result object we call, which much increase its call count.
78         self.result = LoggingDecorator(asserter)
79         asserter.earlier = self.result
80         self.decorated = asserter
81
82     def tearDown(self):
83         # The hook in self.result must have been called
84         self.assertEqual(1, self.result._calls)
85         # The hook in asserter must have been called too, otherwise the
86         # assertion about ordering won't have completed.
87         self.assertEqual(1, self.decorated._calls)
88
89     def test_startTest(self):
90         self.result.startTest(self)
91
92     def test_startTestRun(self):
93         self.result.startTestRun()
94
95     def test_stopTest(self):
96         self.result.stopTest(self)
97
98     def test_stopTestRun(self):
99         self.result.stopTestRun()
100
101     def test_addError(self):
102         self.result.addError(self, subunit.RemoteError())
103
104     def test_addError_details(self):
105         self.result.addError(self, details={})
106
107     def test_addFailure(self):
108         self.result.addFailure(self, subunit.RemoteError())
109
110     def test_addFailure_details(self):
111         self.result.addFailure(self, details={})
112
113     def test_addSuccess(self):
114         self.result.addSuccess(self)
115
116     def test_addSuccess_details(self):
117         self.result.addSuccess(self, details={})
118
119     def test_addSkip(self):
120         self.result.addSkip(self, "foo")
121
122     def test_addSkip_details(self):
123         self.result.addSkip(self, details={})
124
125     def test_addExpectedFailure(self):
126         self.result.addExpectedFailure(self, subunit.RemoteError())
127
128     def test_addExpectedFailure_details(self):
129         self.result.addExpectedFailure(self, details={})
130
131     def test_addUnexpectedSuccess(self):
132         self.result.addUnexpectedSuccess(self)
133
134     def test_addUnexpectedSuccess_details(self):
135         self.result.addUnexpectedSuccess(self, details={})
136
137     def test_progress(self):
138         self.result.progress(1, subunit.PROGRESS_SET)
139
140     def test_wasSuccessful(self):
141         self.result.wasSuccessful()
142
143     def test_shouldStop(self):
144         self.result.shouldStop
145
146     def test_stop(self):
147         self.result.stop()
148
149     def test_time(self):
150         self.result.time(None)
151
152
153 class TestAutoTimingTestResultDecorator(unittest.TestCase):
154
155     def setUp(self):
156         # And end to the chain which captures time events.
157         terminal = TimeCapturingResult()
158         # The result object under test.
159         self.result = subunit.test_results.AutoTimingTestResultDecorator(
160             terminal)
161         self.decorated = terminal
162
163     def test_without_time_calls_time_is_called_and_not_None(self):
164         self.result.startTest(self)
165         self.assertEqual(1, len(self.decorated._calls))
166         self.assertNotEqual(None, self.decorated._calls[0])
167
168     def test_no_time_from_progress(self):
169         self.result.progress(1, subunit.PROGRESS_CUR)
170         self.assertEqual(0, len(self.decorated._calls))
171
172     def test_no_time_from_shouldStop(self):
173         self.decorated.stop()
174         self.result.shouldStop
175         self.assertEqual(0, len(self.decorated._calls))
176
177     def test_calling_time_inhibits_automatic_time(self):
178         # Calling time() outputs a time signal immediately and prevents
179         # automatically adding one when other methods are called.
180         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
181         self.result.time(time)
182         self.result.startTest(self)
183         self.result.stopTest(self)
184         self.assertEqual(1, len(self.decorated._calls))
185         self.assertEqual(time, self.decorated._calls[0])
186
187     def test_calling_time_None_enables_automatic_time(self):
188         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
189         self.result.time(time)
190         self.assertEqual(1, len(self.decorated._calls))
191         self.assertEqual(time, self.decorated._calls[0])
192         # Calling None passes the None through, in case other results care.
193         self.result.time(None)
194         self.assertEqual(2, len(self.decorated._calls))
195         self.assertEqual(None, self.decorated._calls[1])
196         # Calling other methods doesn't generate an automatic time event.
197         self.result.startTest(self)
198         self.assertEqual(3, len(self.decorated._calls))
199         self.assertNotEqual(None, self.decorated._calls[2])
200
201
202 class TestTagCollapsingDecorator(TestCase):
203
204     def test_tags_forwarded_outside_of_tests(self):
205         result = ExtendedTestResult()
206         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
207         tag_collapser.tags(set(['a', 'b']), set())
208         self.assertEquals(
209             [('tags', set(['a', 'b']), set([]))], result._events)
210
211     def test_tags_collapsed_inside_of_tests(self):
212         result = ExtendedTestResult()
213         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
214         test = subunit.RemotedTestCase('foo')
215         tag_collapser.startTest(test)
216         tag_collapser.tags(set(['a']), set())
217         tag_collapser.tags(set(['b']), set(['a']))
218         tag_collapser.tags(set(['c']), set())
219         tag_collapser.stopTest(test)
220         self.assertEquals(
221             [('startTest', test),
222              ('tags', set(['b', 'c']), set(['a'])),
223              ('stopTest', test)],
224             result._events)
225
226     def test_tags_collapsed_inside_of_tests_different_ordering(self):
227         result = ExtendedTestResult()
228         tag_collapser = subunit.test_results.TagCollapsingDecorator(result)
229         test = subunit.RemotedTestCase('foo')
230         tag_collapser.startTest(test)
231         tag_collapser.tags(set(), set(['a']))
232         tag_collapser.tags(set(['a', 'b']), set())
233         tag_collapser.tags(set(['c']), set())
234         tag_collapser.stopTest(test)
235         self.assertEquals(
236             [('startTest', test),
237              ('tags', set(['a', 'b', 'c']), set()),
238              ('stopTest', test)],
239             result._events)
240
241
242 class TestTimeCollapsingDecorator(TestCase):
243
244     def make_time(self):
245         # Heh heh.
246         return datetime.datetime(
247             2000, 1, self.getUniqueInteger(), tzinfo=iso8601.UTC)
248
249     def test_initial_time_forwarded(self):
250         # We always forward the first time event we see.
251         result = ExtendedTestResult()
252         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
253         a_time = self.make_time()
254         tag_collapser.time(a_time)
255         self.assertEquals([('time', a_time)], result._events)
256
257     def test_time_collapsed_to_first_and_last(self):
258         # If there are many consecutive time events, only the first and last
259         # are sent through.
260         result = ExtendedTestResult()
261         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
262         times = [self.make_time() for i in range(5)]
263         for a_time in times:
264             tag_collapser.time(a_time)
265         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
266         self.assertEquals(
267             [('time', times[0]), ('time', times[-1])], result._events[:-1])
268
269     def test_only_one_time_sent(self):
270         # If we receive a single time event followed by a non-time event, we
271         # send exactly one time event.
272         result = ExtendedTestResult()
273         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
274         a_time = self.make_time()
275         tag_collapser.time(a_time)
276         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
277         self.assertEquals([('time', a_time)], result._events[:-1])
278
279     def test_duplicate_times_not_sent(self):
280         # Many time events with the exact same time are collapsed into one
281         # time event.
282         result = ExtendedTestResult()
283         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
284         a_time = self.make_time()
285         for i in range(5):
286             tag_collapser.time(a_time)
287         tag_collapser.startTest(subunit.RemotedTestCase('foo'))
288         self.assertEquals([('time', a_time)], result._events[:-1])
289
290     def test_no_times_inserted(self):
291         result = ExtendedTestResult()
292         tag_collapser = subunit.test_results.TimeCollapsingDecorator(result)
293         a_time = self.make_time()
294         tag_collapser.time(a_time)
295         foo = subunit.RemotedTestCase('foo')
296         tag_collapser.startTest(foo)
297         tag_collapser.addSuccess(foo)
298         tag_collapser.stopTest(foo)
299         self.assertEquals(
300             [('time', a_time),
301              ('startTest', foo),
302              ('addSuccess', foo),
303              ('stopTest', foo)], result._events)
304
305
306 class TestByTestResultTests(testtools.TestCase):
307
308     def setUp(self):
309         super(TestByTestResultTests, self).setUp()
310         self.log = []
311         self.result = subunit.test_results.TestByTestResult(self.on_test)
312         self.result._now = iter(range(5)).next
313
314     def assertCalled(self, **kwargs):
315         defaults = {
316             'test': self,
317             'tags': set(),
318             'details': None,
319             'start_time': 0,
320             'stop_time': 1,
321             }
322         defaults.update(kwargs)
323         self.assertEqual([defaults], self.log)
324
325     def on_test(self, **kwargs):
326         self.log.append(kwargs)
327
328     def test_no_tests_nothing_reported(self):
329         self.result.startTestRun()
330         self.result.stopTestRun()
331         self.assertEqual([], self.log)
332
333     def test_add_success(self):
334         self.result.startTest(self)
335         self.result.addSuccess(self)
336         self.result.stopTest(self)
337         self.assertCalled(status='success')
338
339     def test_add_success_details(self):
340         self.result.startTest(self)
341         details = {'foo': 'bar'}
342         self.result.addSuccess(self, details=details)
343         self.result.stopTest(self)
344         self.assertCalled(status='success', details=details)
345
346     def test_tags(self):
347         if not getattr(self.result, 'tags', None):
348             self.skipTest("No tags in testtools")
349         self.result.tags(['foo'], [])
350         self.result.startTest(self)
351         self.result.addSuccess(self)
352         self.result.stopTest(self)
353         self.assertCalled(status='success', tags=set(['foo']))
354
355     def test_add_error(self):
356         self.result.startTest(self)
357         try:
358             1/0
359         except ZeroDivisionError:
360             error = sys.exc_info()
361         self.result.addError(self, error)
362         self.result.stopTest(self)
363         self.assertCalled(
364             status='error',
365             details={'traceback': TracebackContent(error, self)})
366
367     def test_add_error_details(self):
368         self.result.startTest(self)
369         details = {"foo": text_content("bar")}
370         self.result.addError(self, details=details)
371         self.result.stopTest(self)
372         self.assertCalled(status='error', details=details)
373
374     def test_add_failure(self):
375         self.result.startTest(self)
376         try:
377             self.fail("intentional failure")
378         except self.failureException:
379             failure = sys.exc_info()
380         self.result.addFailure(self, failure)
381         self.result.stopTest(self)
382         self.assertCalled(
383             status='failure',
384             details={'traceback': TracebackContent(failure, self)})
385
386     def test_add_failure_details(self):
387         self.result.startTest(self)
388         details = {"foo": text_content("bar")}
389         self.result.addFailure(self, details=details)
390         self.result.stopTest(self)
391         self.assertCalled(status='failure', details=details)
392
393     def test_add_xfail(self):
394         self.result.startTest(self)
395         try:
396             1/0
397         except ZeroDivisionError:
398             error = sys.exc_info()
399         self.result.addExpectedFailure(self, error)
400         self.result.stopTest(self)
401         self.assertCalled(
402             status='xfail',
403             details={'traceback': TracebackContent(error, self)})
404
405     def test_add_xfail_details(self):
406         self.result.startTest(self)
407         details = {"foo": text_content("bar")}
408         self.result.addExpectedFailure(self, details=details)
409         self.result.stopTest(self)
410         self.assertCalled(status='xfail', details=details)
411
412     def test_add_unexpected_success(self):
413         self.result.startTest(self)
414         details = {'foo': 'bar'}
415         self.result.addUnexpectedSuccess(self, details=details)
416         self.result.stopTest(self)
417         self.assertCalled(status='success', details=details)
418
419     def test_add_skip_reason(self):
420         self.result.startTest(self)
421         reason = self.getUniqueString()
422         self.result.addSkip(self, reason)
423         self.result.stopTest(self)
424         self.assertCalled(
425             status='skip', details={'reason': text_content(reason)})
426
427     def test_add_skip_details(self):
428         self.result.startTest(self)
429         details = {'foo': 'bar'}
430         self.result.addSkip(self, details=details)
431         self.result.stopTest(self)
432         self.assertCalled(status='skip', details=details)
433
434     def test_twice(self):
435         self.result.startTest(self)
436         self.result.addSuccess(self, details={'foo': 'bar'})
437         self.result.stopTest(self)
438         self.result.startTest(self)
439         self.result.addSuccess(self)
440         self.result.stopTest(self)
441         self.assertEqual(
442             [{'test': self,
443               'status': 'success',
444               'start_time': 0,
445               'stop_time': 1,
446               'tags': set(),
447               'details': {'foo': 'bar'}},
448              {'test': self,
449               'status': 'success',
450               'start_time': 2,
451               'stop_time': 3,
452               'tags': set(),
453               'details': None},
454              ],
455             self.log)
456
457
458 class TestCsvResult(testtools.TestCase):
459
460     def test_csv_output(self):
461         stream = StringIO()
462         result = subunit.test_results.csv_result(stream)
463         result._now = iter(range(5)).next
464         result.startTestRun()
465         result.startTest(self)
466         result.addSuccess(self)
467         result.stopTest(self)
468         result.stopTestRun()
469         stream.seek(0)
470         reader = csv.reader(stream)
471         output = list(reader)
472         self.assertEqual(
473             [['test', 'status', 'start_time', 'stop_time'],
474              [self.id(), 'success', '0', '1'],
475              ],
476             output)
477
478
479 def test_suite():
480     loader = subunit.tests.TestUtil.TestLoader()
481     result = loader.loadTestsFromName(__name__)
482     return result