d333c10e390e37a2994fb31cb53fd686866c18b5
[third_party/subunit] / python / subunit / tests / test_test_results.py
1 #
2 #  subunit: extensions to Python unittest to get test results from subprocesses.
3 #  Copyright (C) 2009  Robert Collins <robertc@robertcollins.net>
4 #
5 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6 #  license at the users choice. A copy of both licenses are available in the
7 #  project source as Apache-2.0 and BSD. You may not use this file except in
8 #  compliance with one of these two licences.
9 #  
10 #  Unless required by applicable law or agreed to in writing, software
11 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
13 #  license you chose for the specific language governing permissions and
14 #  limitations under that license.
15 #
16
17 import datetime
18 import unittest
19 from StringIO import StringIO
20 import os
21 import sys
22
23 import subunit
24 import subunit.iso8601 as iso8601
25 import subunit.test_results
26 from subunit.content_type import ContentType
27 from subunit.content import Content
28
29
30 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
31
32     def __init__(self, decorated):
33         self._calls = 0
34         super(LoggingDecorator, self).__init__(decorated)
35
36     def _before_event(self):
37         self._calls += 1
38
39
40 class AssertBeforeTestResult(LoggingDecorator):
41     """A TestResult for checking preconditions."""
42
43     def __init__(self, decorated, test):
44         self.test = test
45         super(AssertBeforeTestResult, self).__init__(decorated)
46
47     def _before_event(self):
48         self.test.assertEqual(1, self.earlier._calls)
49         super(AssertBeforeTestResult, self)._before_event()
50
51
52 class TimeCapturingResult(unittest.TestResult):
53
54     def __init__(self):
55         super(TimeCapturingResult, self).__init__()
56         self._calls = []
57
58     def time(self, a_datetime):
59         self._calls.append(a_datetime)
60
61
62 class LoggingResult(object):
63     """Basic support for logging of results."""
64     
65     def __init__(self):
66         self._calls = []
67
68
69 class Python26TestResult(LoggingResult):
70     """A python 2.6 like test result, that logs."""
71
72     def addError(self, test, err):
73         self._calls.append(('addError', test, err))
74
75     def addFailure(self, test, err):
76         self._calls.append(('addFailure', test, err))
77
78     def addSuccess(self, test):
79         self._calls.append(('addSuccess', test))
80
81     def startTest(self, test):
82         self._calls.append(('startTest', test))
83
84     def stopTest(self, test):
85         self._calls.append(('stopTest', test))
86
87
88 class Python27TestResult(Python26TestResult):
89     """A python 2.7 like test result, that logs."""
90
91     def addExpectedFailure(self, test, err):
92         self._calls.append(('addExpectedFailure', test, err))
93
94     def addSkip(self, test, reason):
95         self._calls.append(('addSkip', test, reason))
96
97     def addUnexpectedSuccess(self, test):
98         self._calls.append(('addUnexpectedSuccess', test))
99
100     def startTestRun(self):
101         self._calls.append(('startTestRun',))
102
103     def stopTestRun(self):
104         self._calls.append(('stopTestRun',))
105
106
107 class ExtendedTestResult(Python27TestResult):
108     """A test result like the proposed extended unittest result API."""
109
110     def addError(self, test, err=None, details=None):
111         self._calls.append(('addError', test, err or details))
112
113     def addFailure(self, test, err=None, details=None):
114         self._calls.append(('addFailure', test, err or details))
115
116     def addExpectedFailure(self, test, err=None, details=None):
117         self._calls.append(('addExpectedFailure', test, err or details))
118
119     def addSkip(self, test, reason=None, details=None):
120         self._calls.append(('addSkip', test, reason or details))
121
122     def addSuccess(self, test, details=None):
123         if details:
124             self._calls.append(('addSuccess', test, details))
125         else:
126             self._calls.append(('addSuccess', test))
127
128     def addUnexpectedSuccess(self, test, details=None):
129         if details:
130             self._calls.append(('addUnexpectedSuccess', test, details))
131         else:
132             self._calls.append(('addUnexpectedSuccess', test))
133
134     def progress(self, offset, whence):
135         self._calls.append(('progress', offset, whence))
136
137     def tags(self, new_tags, gone_tags):
138         self._calls.append(('tags', new_tags, gone_tags))
139
140     def time(self, time):
141         self._calls.append(('time', time))
142
143
144 class TestExtendedToOriginalResultDecoratorBase(unittest.TestCase):
145
146     def make_26_result(self):
147         self.result = Python26TestResult()
148         self.make_converter()
149
150     def make_27_result(self):
151         self.result = Python27TestResult()
152         self.make_converter()
153
154     def make_converter(self):
155         self.converter = \
156             subunit.test_results.ExtendedToOriginalDecorator(self.result)
157
158     def make_extended_result(self):
159         self.result = ExtendedTestResult()
160         self.make_converter()
161
162     def check_outcome_details(self, outcome):
163         """Call an outcome with a details dict to be passed through."""
164         # This dict is /not/ convertible - thats deliberate, as it should
165         # not hit the conversion code path.
166         details = {'foo': 'bar'}
167         getattr(self.converter, outcome)(self, details=details)
168         self.assertEqual([(outcome, self, details)], self.result._calls)
169
170     def get_details_and_string(self):
171         """Get a details dict and expected string."""
172         text1 = lambda:["1\n2\n"]
173         text2 = lambda:["3\n4\n"]
174         bin1 = lambda:["5\n"]
175         details = {'text 1': Content(ContentType('text', 'plain'), text1),
176             'text 2': Content(ContentType('text', 'strange'), text2),
177             'bin 1': Content(ContentType('application', 'binary'), bin1)}
178         return (details, "Binary content: bin 1\n"
179             "Text attachment: text 1\n------------\n1\n2\n"
180             "------------\nText attachment: text 2\n------------\n"
181             "3\n4\n------------\n")
182
183     def check_outcome_details_to_exec_info(self, outcome, expected=None):
184         """Call an outcome with a details dict to be made into exc_info."""
185         # The conversion is a done using RemoteError and the string contents
186         # of the text types in the details dict.
187         if not expected:
188             expected = outcome
189         details, err_str = self.get_details_and_string()
190         getattr(self.converter, outcome)(self, details=details)
191         err = subunit.RemoteError(err_str)
192         self.assertEqual([(expected, self, err)], self.result._calls)
193
194     def check_outcome_details_to_nothing(self, outcome, expected=None):
195         """Call an outcome with a details dict to be swallowed."""
196         if not expected:
197             expected = outcome
198         details = {'foo': 'bar'}
199         getattr(self.converter, outcome)(self, details=details)
200         self.assertEqual([(expected, self)], self.result._calls)
201
202     def check_outcome_details_to_string(self, outcome):
203         """Call an outcome with a details dict to be stringified."""
204         details, err_str = self.get_details_and_string()
205         getattr(self.converter, outcome)(self, details=details)
206         self.assertEqual([(outcome, self, err_str)], self.result._calls)
207
208     def check_outcome_exc_info(self, outcome, expected=None):
209         """Check that calling a legacy outcome still works."""
210         # calling some outcome with the legacy exc_info style api (no keyword
211         # parameters) gets passed through.
212         if not expected:
213             expected = outcome
214         err = subunit.RemoteError("foo\nbar\n")
215         getattr(self.converter, outcome)(self, err)
216         self.assertEqual([(expected, self, err)], self.result._calls)
217
218     def check_outcome_exc_info_to_nothing(self, outcome, expected=None):
219         """Check that calling a legacy outcome on a fallback works."""
220         # calling some outcome with the legacy exc_info style api (no keyword
221         # parameters) gets passed through.
222         if not expected:
223             expected = outcome
224         err = subunit.RemoteError("foo\nbar\n")
225         getattr(self.converter, outcome)(self, err)
226         self.assertEqual([(expected, self)], self.result._calls)
227
228     def check_outcome_nothing(self, outcome, expected=None):
229         """Check that calling a legacy outcome still works."""
230         if not expected:
231             expected = outcome
232         getattr(self.converter, outcome)(self)
233         self.assertEqual([(expected, self)], self.result._calls)
234
235     def check_outcome_string_nothing(self, outcome, expected):
236         """Check that calling outcome with a string calls expected."""
237         getattr(self.converter, outcome)(self, "foo")
238         self.assertEqual([(expected, self)], self.result._calls)
239
240     def check_outcome_string(self, outcome):
241         """Check that calling outcome with a string works."""
242         getattr(self.converter, outcome)(self, "foo")
243         self.assertEqual([(outcome, self, "foo")], self.result._calls)
244
245
246 class TestExtendedToOriginalResultDecorator(
247     TestExtendedToOriginalResultDecoratorBase):
248
249     def test_progress_py26(self):
250         self.make_26_result()
251         self.converter.progress(1, 2)
252
253     def test_progress_py27(self):
254         self.make_27_result()
255         self.converter.progress(1, 2)
256
257     def test_progress_pyextended(self):
258         self.make_extended_result()
259         self.converter.progress(1, 2)
260         self.assertEqual([('progress', 1, 2)], self.result._calls)
261
262     def test_startTest_py26(self):
263         self.make_26_result()
264         self.converter.startTest(self)
265         self.assertEqual([('startTest', self)], self.result._calls)
266     
267     def test_startTest_py27(self):
268         self.make_27_result()
269         self.converter.startTest(self)
270         self.assertEqual([('startTest', self)], self.result._calls)
271
272     def test_startTest_pyextended(self):
273         self.make_extended_result()
274         self.converter.startTest(self)
275         self.assertEqual([('startTest', self)], self.result._calls)
276
277     def test_startTestRun_py26(self):
278         self.make_26_result()
279         self.converter.startTestRun()
280         self.assertEqual([], self.result._calls)
281     
282     def test_startTestRun_py27(self):
283         self.make_27_result()
284         self.converter.startTestRun()
285         self.assertEqual([('startTestRun',)], self.result._calls)
286
287     def test_startTestRun_pyextended(self):
288         self.make_extended_result()
289         self.converter.startTestRun()
290         self.assertEqual([('startTestRun',)], self.result._calls)
291
292     def test_stopTest_py26(self):
293         self.make_26_result()
294         self.converter.stopTest(self)
295         self.assertEqual([('stopTest', self)], self.result._calls)
296     
297     def test_stopTest_py27(self):
298         self.make_27_result()
299         self.converter.stopTest(self)
300         self.assertEqual([('stopTest', self)], self.result._calls)
301
302     def test_stopTest_pyextended(self):
303         self.make_extended_result()
304         self.converter.stopTest(self)
305         self.assertEqual([('stopTest', self)], self.result._calls)
306
307     def test_stopTestRun_py26(self):
308         self.make_26_result()
309         self.converter.stopTestRun()
310         self.assertEqual([], self.result._calls)
311     
312     def test_stopTestRun_py27(self):
313         self.make_27_result()
314         self.converter.stopTestRun()
315         self.assertEqual([('stopTestRun',)], self.result._calls)
316
317     def test_stopTestRun_pyextended(self):
318         self.make_extended_result()
319         self.converter.stopTestRun()
320         self.assertEqual([('stopTestRun',)], self.result._calls)
321
322     def test_tags_py26(self):
323         self.make_26_result()
324         self.converter.tags(1, 2)
325
326     def test_tags_py27(self):
327         self.make_27_result()
328         self.converter.tags(1, 2)
329
330     def test_tags_pyextended(self):
331         self.make_extended_result()
332         self.converter.tags(1, 2)
333         self.assertEqual([('tags', 1, 2)], self.result._calls)
334
335     def test_time_py26(self):
336         self.make_26_result()
337         self.converter.time(1)
338
339     def test_time_py27(self):
340         self.make_27_result()
341         self.converter.time(1)
342
343     def test_time_pyextended(self):
344         self.make_extended_result()
345         self.converter.time(1)
346         self.assertEqual([('time', 1)], self.result._calls)
347
348
349 class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase):
350
351     outcome = 'addError'
352
353     def test_outcome_Original_py26(self):
354         self.make_26_result()
355         self.check_outcome_exc_info(self.outcome)
356     
357     def test_outcome_Original_py27(self):
358         self.make_27_result()
359         self.check_outcome_exc_info(self.outcome)
360
361     def test_outcome_Original_pyextended(self):
362         self.make_extended_result()
363         self.check_outcome_exc_info(self.outcome)
364
365     def test_outcome_Extended_py26(self):
366         self.make_26_result()
367         self.check_outcome_details_to_exec_info(self.outcome)
368     
369     def test_outcome_Extended_py27(self):
370         self.make_27_result()
371         self.check_outcome_details_to_exec_info(self.outcome)
372
373     def test_outcome_Extended_pyextended(self):
374         self.make_extended_result()
375         self.check_outcome_details(self.outcome)
376
377     def test_outcome__no_details(self):
378         self.make_extended_result()
379         self.assertRaises(ValueError,
380             getattr(self.converter, self.outcome), self)
381
382
383 class TestExtendedToOriginalAddFailure(
384     TestExtendedToOriginalAddError):
385
386     outcome = 'addFailure'
387
388
389 class TestExtendedToOriginalAddExpectedFailure(
390     TestExtendedToOriginalAddError):
391
392     outcome = 'addExpectedFailure'
393
394     def test_outcome_Original_py26(self):
395         self.make_26_result()
396         self.check_outcome_exc_info_to_nothing(self.outcome, 'addSuccess')
397     
398     def test_outcome_Extended_py26(self):
399         self.make_26_result()
400         self.check_outcome_details_to_nothing(self.outcome, 'addSuccess')
401     
402
403
404 class TestExtendedToOriginalAddSkip(
405     TestExtendedToOriginalResultDecoratorBase):
406
407     outcome = 'addSkip'
408
409     def test_outcome_Original_py26(self):
410         self.make_26_result()
411         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
412     
413     def test_outcome_Original_py27(self):
414         self.make_27_result()
415         self.check_outcome_string(self.outcome)
416
417     def test_outcome_Original_pyextended(self):
418         self.make_extended_result()
419         self.check_outcome_string(self.outcome)
420
421     def test_outcome_Extended_py26(self):
422         self.make_26_result()
423         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
424     
425     def test_outcome_Extended_py27(self):
426         self.make_27_result()
427         self.check_outcome_details_to_string(self.outcome)
428
429     def test_outcome_Extended_pyextended(self):
430         self.make_extended_result()
431         self.check_outcome_details(self.outcome)
432
433     def test_outcome__no_details(self):
434         self.make_extended_result()
435         self.assertRaises(ValueError,
436             getattr(self.converter, self.outcome), self)
437
438
439 class TestExtendedToOriginalAddSuccess(
440     TestExtendedToOriginalResultDecoratorBase):
441
442     outcome = 'addSuccess'
443     expected = 'addSuccess'
444
445     def test_outcome_Original_py26(self):
446         self.make_26_result()
447         self.check_outcome_nothing(self.outcome, self.expected)
448     
449     def test_outcome_Original_py27(self):
450         self.make_27_result()
451         self.check_outcome_nothing(self.outcome)
452
453     def test_outcome_Original_pyextended(self):
454         self.make_extended_result()
455         self.check_outcome_nothing(self.outcome)
456
457     def test_outcome_Extended_py26(self):
458         self.make_26_result()
459         self.check_outcome_details_to_nothing(self.outcome, self.expected)
460     
461     def test_outcome_Extended_py27(self):
462         self.make_27_result()
463         self.check_outcome_details_to_nothing(self.outcome)
464
465     def test_outcome_Extended_pyextended(self):
466         self.make_extended_result()
467         self.check_outcome_details(self.outcome)
468
469
470 class TestExtendedToOriginalAddUnexpectedSuccess(
471     TestExtendedToOriginalAddSuccess):
472
473     outcome = 'addUnexpectedSuccess'
474
475
476 class TestHookedTestResultDecorator(unittest.TestCase):
477
478     def setUp(self):
479         # And end to the chain
480         terminal = unittest.TestResult()
481         # Asserts that the call was made to self.result before asserter was
482         # called.
483         asserter = AssertBeforeTestResult(terminal, self)
484         # The result object we call, which much increase its call count.
485         self.result = LoggingDecorator(asserter)
486         asserter.earlier = self.result
487
488     def tearDown(self):
489         # The hook in self.result must have been called
490         self.assertEqual(1, self.result._calls)
491         # The hook in asserter must have been called too, otherwise the
492         # assertion about ordering won't have completed.
493         self.assertEqual(1, self.result.decorated._calls)
494
495     def test_startTest(self):
496         self.result.startTest(self)
497         
498     def test_startTestRun(self):
499         self.result.startTestRun()
500         
501     def test_stopTest(self):
502         self.result.stopTest(self)
503         
504     def test_stopTestRun(self):
505         self.result.stopTestRun()
506
507     def test_addError(self):
508         self.result.addError(self, subunit.RemoteError())
509         
510     def test_addFailure(self):
511         self.result.addFailure(self, subunit.RemoteError())
512
513     def test_addSuccess(self):
514         self.result.addSuccess(self)
515
516     def test_addSkip(self):
517         self.result.addSkip(self, "foo")
518
519     def test_addExpectedFailure(self):
520         self.result.addExpectedFailure(self, subunit.RemoteError())
521
522     def test_addUnexpectedSuccess(self):
523         self.result.addUnexpectedSuccess(self)
524
525     def test_progress(self):
526         self.result.progress(1, subunit.PROGRESS_SET)
527
528     def test_wasSuccessful(self):
529         self.result.wasSuccessful()
530
531     def test_shouldStop(self):
532         self.result.shouldStop
533
534     def test_stop(self):
535         self.result.stop()
536
537     def test_time(self):
538         self.result.time(None)
539  
540
541 class TestAutoTimingTestResultDecorator(unittest.TestCase):
542
543     def setUp(self):
544         # And end to the chain which captures time events.
545         terminal = TimeCapturingResult()
546         # The result object under test.
547         self.result = subunit.test_results.AutoTimingTestResultDecorator(
548             terminal)
549
550     def test_without_time_calls_time_is_called_and_not_None(self):
551         self.result.startTest(self)
552         self.assertEqual(1, len(self.result.decorated._calls))
553         self.assertNotEqual(None, self.result.decorated._calls[0])
554
555     def test_no_time_from_progress(self):
556         self.result.progress(1, subunit.PROGRESS_CUR)
557         self.assertEqual(0, len(self.result.decorated._calls))
558
559     def test_no_time_from_shouldStop(self):
560         self.result.decorated.stop()
561         self.result.shouldStop
562         self.assertEqual(0, len(self.result.decorated._calls))
563
564     def test_calling_time_inhibits_automatic_time(self):
565         # Calling time() outputs a time signal immediately and prevents
566         # automatically adding one when other methods are called.
567         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
568         self.result.time(time)
569         self.result.startTest(self)
570         self.result.stopTest(self)
571         self.assertEqual(1, len(self.result.decorated._calls))
572         self.assertEqual(time, self.result.decorated._calls[0])
573
574     def test_calling_time_None_enables_automatic_time(self):
575         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
576         self.result.time(time)
577         self.assertEqual(1, len(self.result.decorated._calls))
578         self.assertEqual(time, self.result.decorated._calls[0])
579         # Calling None passes the None through, in case other results care.
580         self.result.time(None)
581         self.assertEqual(2, len(self.result.decorated._calls))
582         self.assertEqual(None, self.result.decorated._calls[1])
583         # Calling other methods doesn't generate an automatic time event.
584         self.result.startTest(self)
585         self.assertEqual(3, len(self.result.decorated._calls))
586         self.assertNotEqual(None, self.result.decorated._calls[2])
587
588
589 def test_suite():
590     loader = subunit.tests.TestUtil.TestLoader()
591     result = loader.loadTestsFromName(__name__)
592     return result