git.samba.org - third_party/subunit/blob - python/subunit/tests/test_test_results.py

   1 #
   2 #  subunit: extensions to Python unittest to get test results from subprocesses.
   3 #  Copyright (C) 2009  Robert Collins <robertc@robertcollins.net>
   4 #
   5 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
   6 #  license at the users choice. A copy of both licenses are available in the
   7 #  project source as Apache-2.0 and BSD. You may not use this file except in
   8 #  compliance with one of these two licences.
   9 #
  10 #  Unless required by applicable law or agreed to in writing, software
  11 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
  12 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
  13 #  license you chose for the specific language governing permissions and
  14 #  limitations under that license.
  15 #
  16
  17 import datetime
  18 import unittest
  19 from StringIO import StringIO
  20 import os
  21 import sys
  22
  23 import subunit
  24 import subunit.iso8601 as iso8601
  25 import subunit.test_results
  26 from subunit.content_type import ContentType
  27 from subunit.content import Content
  28
  29
  30 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
  31
  32     def __init__(self, decorated):
  33         self._calls = 0
  34         super(LoggingDecorator, self).__init__(decorated)
  35
  36     def _before_event(self):
  37         self._calls += 1
  38
  39
  40 class AssertBeforeTestResult(LoggingDecorator):
  41     """A TestResult for checking preconditions."""
  42
  43     def __init__(self, decorated, test):
  44         self.test = test
  45         super(AssertBeforeTestResult, self).__init__(decorated)
  46
  47     def _before_event(self):
  48         self.test.assertEqual(1, self.earlier._calls)
  49         super(AssertBeforeTestResult, self)._before_event()
  50
  51
  52 class TimeCapturingResult(unittest.TestResult):
  53
  54     def __init__(self):
  55         super(TimeCapturingResult, self).__init__()
  56         self._calls = []
  57
  58     def time(self, a_datetime):
  59         self._calls.append(a_datetime)
  60
  61
  62 class LoggingResult(object):
  63     """Basic support for logging of results."""
  64
  65     def __init__(self):
  66         self._calls = []
  67
  68
  69 class Python26TestResult(LoggingResult):
  70     """A python 2.6 like test result, that logs."""
  71
  72     def addError(self, test, err):
  73         self._calls.append(('addError', test, err))
  74
  75     def addFailure(self, test, err):
  76         self._calls.append(('addFailure', test, err))
  77
  78     def addSuccess(self, test):
  79         self._calls.append(('addSuccess', test))
  80
  81     def startTest(self, test):
  82         self._calls.append(('startTest', test))
  83
  84     def stopTest(self, test):
  85         self._calls.append(('stopTest', test))
  86
  87
  88 class Python27TestResult(Python26TestResult):
  89     """A python 2.7 like test result, that logs."""
  90
  91     def addExpectedFailure(self, test, err):
  92         self._calls.append(('addExpectedFailure', test, err))
  93
  94     def addSkip(self, test, reason):
  95         self._calls.append(('addSkip', test, reason))
  96
  97     def addUnexpectedSuccess(self, test):
  98         self._calls.append(('addUnexpectedSuccess', test))
  99
 100     def startTestRun(self):
 101         self._calls.append(('startTestRun',))
 102
 103     def stopTestRun(self):
 104         self._calls.append(('stopTestRun',))
 105
 106
 107 class ExtendedTestResult(Python27TestResult):
 108     """A test result like the proposed extended unittest result API."""
 109
 110     def addError(self, test, err=None, details=None):
 111         self._calls.append(('addError', test, err or details))
 112
 113     def addFailure(self, test, err=None, details=None):
 114         self._calls.append(('addFailure', test, err or details))
 115
 116     def addExpectedFailure(self, test, err=None, details=None):
 117         self._calls.append(('addExpectedFailure', test, err or details))
 118
 119     def addSkip(self, test, reason=None, details=None):
 120         self._calls.append(('addSkip', test, reason or details))
 121
 122     def addSuccess(self, test, details=None):
 123         if details:
 124             self._calls.append(('addSuccess', test, details))
 125         else:
 126             self._calls.append(('addSuccess', test))
 127
 128     def addUnexpectedSuccess(self, test, details=None):
 129         if details:
 130             self._calls.append(('addUnexpectedSuccess', test, details))
 131         else:
 132             self._calls.append(('addUnexpectedSuccess', test))
 133
 134
 135 class TestExtendedToOriginalResultDecoratorBase(unittest.TestCase):
 136
 137     def make_26_result(self):
 138         self.result = Python26TestResult()
 139         self.make_converter()
 140
 141     def make_27_result(self):
 142         self.result = Python27TestResult()
 143         self.make_converter()
 144
 145     def make_converter(self):
 146         self.converter = \
 147             subunit.test_results.ExtendedToOriginalDecorator(self.result)
 148
 149     def make_extended_result(self):
 150         self.result = ExtendedTestResult()
 151         self.make_converter()
 152
 153     def check_outcome_details(self, outcome):
 154         """Call an outcome with a details dict to be passed through."""
 155         # This dict is /not/ convertible - thats deliberate, as it should
 156         # not hit the conversion code path.
 157         details = {'foo': 'bar'}
 158         getattr(self.converter, outcome)(self, details=details)
 159         self.assertEqual([(outcome, self, details)], self.result._calls)
 160
 161     def get_details_and_string(self):
 162         """Get a details dict and expected string."""
 163         text1 = lambda:["1\n2\n"]
 164         text2 = lambda:["3\n4\n"]
 165         bin1 = lambda:["5\n"]
 166         details = {'text 1': Content(ContentType('text', 'plain'), text1),
 167             'text 2': Content(ContentType('text', 'strange'), text2),
 168             'bin 1': Content(ContentType('application', 'binary'), bin1)}
 169         return (details, "Binary content: bin 1\n"
 170             "Text attachment: text 1\n------------\n1\n2\n"
 171             "------------\nText attachment: text 2\n------------\n"
 172             "3\n4\n------------\n")
 173
 174     def check_outcome_details_to_exec_info(self, outcome, expected=None):
 175         """Call an outcome with a details dict to be made into exc_info."""
 176         # The conversion is a done using RemoteError and the string contents
 177         # of the text types in the details dict.
 178         if not expected:
 179             expected = outcome
 180         details, err_str = self.get_details_and_string()
 181         getattr(self.converter, outcome)(self, details=details)
 182         err = subunit.RemoteError(err_str)
 183         self.assertEqual([(expected, self, err)], self.result._calls)
 184
 185     def check_outcome_details_to_nothing(self, outcome, expected=None):
 186         """Call an outcome with a details dict to be swallowed."""
 187         if not expected:
 188             expected = outcome
 189         details = {'foo': 'bar'}
 190         getattr(self.converter, outcome)(self, details=details)
 191         self.assertEqual([(expected, self)], self.result._calls)
 192
 193     def check_outcome_details_to_string(self, outcome):
 194         """Call an outcome with a details dict to be stringified."""
 195         details, err_str = self.get_details_and_string()
 196         getattr(self.converter, outcome)(self, details=details)
 197         self.assertEqual([(outcome, self, err_str)], self.result._calls)
 198
 199     def check_outcome_exc_info(self, outcome, expected=None):
 200         """Check that calling a legacy outcome still works."""
 201         # calling some outcome with the legacy exc_info style api (no keyword
 202         # parameters) gets passed through.
 203         if not expected:
 204             expected = outcome
 205         err = subunit.RemoteError("foo\nbar\n")
 206         getattr(self.converter, outcome)(self, err)
 207         self.assertEqual([(expected, self, err)], self.result._calls)
 208
 209     def check_outcome_nothing(self, outcome, expected=None):
 210         """Check that calling a legacy outcome still works."""
 211         if not expected:
 212             expected = outcome
 213         getattr(self.converter, outcome)(self)
 214         self.assertEqual([(expected, self)], self.result._calls)
 215
 216     def check_outcome_string_nothing(self, outcome, expected):
 217         """Check that calling outcome with a string calls expected."""
 218         getattr(self.converter, outcome)(self, "foo")
 219         self.assertEqual([(expected, self)], self.result._calls)
 220
 221     def check_outcome_string(self, outcome):
 222         """Check that calling outcome with a string works."""
 223         getattr(self.converter, outcome)(self, "foo")
 224         self.assertEqual([(outcome, self, "foo")], self.result._calls)
 225
 226
 227 class TestExtendedToOriginalResultDecorator(
 228     TestExtendedToOriginalResultDecoratorBase):
 229
 230     def test_startTest_py26(self):
 231         self.make_26_result()
 232         self.converter.startTest(self)
 233         self.assertEqual([('startTest', self)], self.result._calls)
 234
 235     def test_startTest_py27(self):
 236         self.make_27_result()
 237         self.converter.startTest(self)
 238         self.assertEqual([('startTest', self)], self.result._calls)
 239
 240     def test_startTest_pyextended(self):
 241         self.make_extended_result()
 242         self.converter.startTest(self)
 243         self.assertEqual([('startTest', self)], self.result._calls)
 244
 245     def test_startTestRun_py26(self):
 246         self.make_26_result()
 247         self.converter.startTestRun()
 248         self.assertEqual([], self.result._calls)
 249
 250     def test_startTestRun_py27(self):
 251         self.make_27_result()
 252         self.converter.startTestRun()
 253         self.assertEqual([('startTestRun',)], self.result._calls)
 254
 255     def test_startTestRun_pyextended(self):
 256         self.make_extended_result()
 257         self.converter.startTestRun()
 258         self.assertEqual([('startTestRun',)], self.result._calls)
 259
 260     def test_stopTest_py26(self):
 261         self.make_26_result()
 262         self.converter.stopTest(self)
 263         self.assertEqual([('stopTest', self)], self.result._calls)
 264
 265     def test_stopTest_py27(self):
 266         self.make_27_result()
 267         self.converter.stopTest(self)
 268         self.assertEqual([('stopTest', self)], self.result._calls)
 269
 270     def test_stopTest_pyextended(self):
 271         self.make_extended_result()
 272         self.converter.stopTest(self)
 273         self.assertEqual([('stopTest', self)], self.result._calls)
 274
 275     def test_stopTestRun_py26(self):
 276         self.make_26_result()
 277         self.converter.stopTestRun()
 278         self.assertEqual([], self.result._calls)
 279
 280     def test_stopTestRun_py27(self):
 281         self.make_27_result()
 282         self.converter.stopTestRun()
 283         self.assertEqual([('stopTestRun',)], self.result._calls)
 284
 285     def test_stopTestRun_pyextended(self):
 286         self.make_extended_result()
 287         self.converter.stopTestRun()
 288         self.assertEqual([('stopTestRun',)], self.result._calls)
 289
 290
 291 class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase):
 292
 293     outcome = 'addError'
 294
 295     def test_outcome_Original_py26(self):
 296         self.make_26_result()
 297         self.check_outcome_exc_info(self.outcome)
 298
 299     def test_outcome_Original_py27(self):
 300         self.make_27_result()
 301         self.check_outcome_exc_info(self.outcome)
 302
 303     def test_outcome_Original_pyextended(self):
 304         self.make_extended_result()
 305         self.check_outcome_exc_info(self.outcome)
 306
 307     def test_outcome_Extended_py26(self):
 308         self.make_26_result()
 309         self.check_outcome_details_to_exec_info(self.outcome)
 310
 311     def test_outcome_Extended_py27(self):
 312         self.make_27_result()
 313         self.check_outcome_details_to_exec_info(self.outcome)
 314
 315     def test_outcome_Extended_pyextended(self):
 316         self.make_extended_result()
 317         self.check_outcome_details(self.outcome)
 318
 319     def test_outcome__no_details(self):
 320         self.make_extended_result()
 321         self.assertRaises(ValueError,
 322             getattr(self.converter, self.outcome), self)
 323
 324
 325 class TestExtendedToOriginalAddFailure(
 326     TestExtendedToOriginalAddError):
 327
 328     outcome = 'addFailure'
 329
 330
 331 class TestExtendedToOriginalAddExpectedFailure(
 332     TestExtendedToOriginalAddError):
 333
 334     outcome = 'addExpectedFailure'
 335
 336     def test_outcome_Original_py26(self):
 337         self.make_26_result()
 338         self.check_outcome_exc_info(self.outcome, 'addFailure')
 339
 340     def test_outcome_Extended_py26(self):
 341         self.make_26_result()
 342         self.check_outcome_details_to_exec_info(self.outcome, 'addFailure')
 343
 344
 345
 346 class TestExtendedToOriginalAddSkip(
 347     TestExtendedToOriginalResultDecoratorBase):
 348
 349     outcome = 'addSkip'
 350
 351     def test_outcome_Original_py26(self):
 352         self.make_26_result()
 353         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
 354
 355     def test_outcome_Original_py27(self):
 356         self.make_27_result()
 357         self.check_outcome_string(self.outcome)
 358
 359     def test_outcome_Original_pyextended(self):
 360         self.make_extended_result()
 361         self.check_outcome_string(self.outcome)
 362
 363     def test_outcome_Extended_py26(self):
 364         self.make_26_result()
 365         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
 366
 367     def test_outcome_Extended_py27(self):
 368         self.make_27_result()
 369         self.check_outcome_details_to_string(self.outcome)
 370
 371     def test_outcome_Extended_pyextended(self):
 372         self.make_extended_result()
 373         self.check_outcome_details(self.outcome)
 374
 375     def test_outcome__no_details(self):
 376         self.make_extended_result()
 377         self.assertRaises(ValueError,
 378             getattr(self.converter, self.outcome), self)
 379
 380
 381 class TestExtendedToOriginalAddSuccess(
 382     TestExtendedToOriginalResultDecoratorBase):
 383
 384     outcome = 'addSuccess'
 385     expected = 'addSuccess'
 386
 387     def test_outcome_Original_py26(self):
 388         self.make_26_result()
 389         self.check_outcome_nothing(self.outcome, self.expected)
 390
 391     def test_outcome_Original_py27(self):
 392         self.make_27_result()
 393         self.check_outcome_nothing(self.outcome)
 394
 395     def test_outcome_Original_pyextended(self):
 396         self.make_extended_result()
 397         self.check_outcome_nothing(self.outcome)
 398
 399     def test_outcome_Extended_py26(self):
 400         self.make_26_result()
 401         self.check_outcome_details_to_nothing(self.outcome, self.expected)
 402
 403     def test_outcome_Extended_py27(self):
 404         self.make_27_result()
 405         self.check_outcome_details_to_nothing(self.outcome)
 406
 407     def test_outcome_Extended_pyextended(self):
 408         self.make_extended_result()
 409         self.check_outcome_details(self.outcome)
 410
 411
 412 class TestExtendedToOriginalAddUnexpectedSuccess(
 413     TestExtendedToOriginalAddSuccess):
 414
 415     outcome = 'addUnexpectedSuccess'
 416
 417
 418 class TestHookedTestResultDecorator(unittest.TestCase):
 419
 420     def setUp(self):
 421         # And end to the chain
 422         terminal = unittest.TestResult()
 423         # Asserts that the call was made to self.result before asserter was
 424         # called.
 425         asserter = AssertBeforeTestResult(terminal, self)
 426         # The result object we call, which much increase its call count.
 427         self.result = LoggingDecorator(asserter)
 428         asserter.earlier = self.result
 429
 430     def tearDown(self):
 431         # The hook in self.result must have been called
 432         self.assertEqual(1, self.result._calls)
 433         # The hook in asserter must have been called too, otherwise the
 434         # assertion about ordering won't have completed.
 435         self.assertEqual(1, self.result.decorated._calls)
 436
 437     def test_startTest(self):
 438         self.result.startTest(self)
 439
 440     def test_startTestRun(self):
 441         self.result.startTestRun()
 442
 443     def test_stopTest(self):
 444         self.result.stopTest(self)
 445
 446     def test_stopTestRun(self):
 447         self.result.stopTestRun()
 448
 449     def test_addError(self):
 450         self.result.addError(self, subunit.RemoteError())
 451
 452     def test_addFailure(self):
 453         self.result.addFailure(self, subunit.RemoteError())
 454
 455     def test_addSuccess(self):
 456         self.result.addSuccess(self)
 457
 458     def test_addSkip(self):
 459         self.result.addSkip(self, "foo")
 460
 461     def test_addExpectedFailure(self):
 462         self.result.addExpectedFailure(self, subunit.RemoteError())
 463
 464     def test_addUnexpectedSuccess(self):
 465         self.result.addUnexpectedSuccess(self)
 466
 467     def test_progress(self):
 468         self.result.progress(1, subunit.PROGRESS_SET)
 469
 470     def test_wasSuccessful(self):
 471         self.result.wasSuccessful()
 472
 473     def test_shouldStop(self):
 474         self.result.shouldStop
 475
 476     def test_stop(self):
 477         self.result.stop()
 478
 479     def test_time(self):
 480         self.result.time(None)
 481
 482
 483 class TestAutoTimingTestResultDecorator(unittest.TestCase):
 484
 485     def setUp(self):
 486         # And end to the chain which captures time events.
 487         terminal = TimeCapturingResult()
 488         # The result object under test.
 489         self.result = subunit.test_results.AutoTimingTestResultDecorator(
 490             terminal)
 491
 492     def test_without_time_calls_time_is_called_and_not_None(self):
 493         self.result.startTest(self)
 494         self.assertEqual(1, len(self.result.decorated._calls))
 495         self.assertNotEqual(None, self.result.decorated._calls[0])
 496
 497     def test_no_time_from_progress(self):
 498         self.result.progress(1, subunit.PROGRESS_CUR)
 499         self.assertEqual(0, len(self.result.decorated._calls))
 500
 501     def test_no_time_from_shouldStop(self):
 502         self.result.decorated.stop()
 503         self.result.shouldStop
 504         self.assertEqual(0, len(self.result.decorated._calls))
 505
 506     def test_calling_time_inhibits_automatic_time(self):
 507         # Calling time() outputs a time signal immediately and prevents
 508         # automatically adding one when other methods are called.
 509         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 510         self.result.time(time)
 511         self.result.startTest(self)
 512         self.result.stopTest(self)
 513         self.assertEqual(1, len(self.result.decorated._calls))
 514         self.assertEqual(time, self.result.decorated._calls[0])
 515
 516     def test_calling_time_None_enables_automatic_time(self):
 517         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 518         self.result.time(time)
 519         self.assertEqual(1, len(self.result.decorated._calls))
 520         self.assertEqual(time, self.result.decorated._calls[0])
 521         # Calling None passes the None through, in case other results care.
 522         self.result.time(None)
 523         self.assertEqual(2, len(self.result.decorated._calls))
 524         self.assertEqual(None, self.result.decorated._calls[1])
 525         # Calling other methods doesn't generate an automatic time event.
 526         self.result.startTest(self)
 527         self.assertEqual(3, len(self.result.decorated._calls))
 528         self.assertNotEqual(None, self.result.decorated._calls[2])
 529
 530
 531 def test_suite():
 532     loader = subunit.tests.TestUtil.TestLoader()
 533     result = loader.loadTestsFromName(__name__)
 534     return result