git.samba.org - third_party/subunit/blob - python/subunit/tests/test_test_results.py

   1 #
   2 #  subunit: extensions to Python unittest to get test results from subprocesses.
   3 #  Copyright (C) 2009  Robert Collins <robertc@robertcollins.net>
   4 #
   5 #  Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
   6 #  license at the users choice. A copy of both licenses are available in the
   7 #  project source as Apache-2.0 and BSD. You may not use this file except in
   8 #  compliance with one of these two licences.
   9 #
  10 #  Unless required by applicable law or agreed to in writing, software
  11 #  distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
  12 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
  13 #  license you chose for the specific language governing permissions and
  14 #  limitations under that license.
  15 #
  16
  17 import datetime
  18 import unittest
  19 from StringIO import StringIO
  20 import os
  21 import sys
  22
  23 import subunit
  24 import subunit.iso8601 as iso8601
  25 import subunit.test_results
  26 from subunit.content_type import ContentType
  27 from subunit.content import Content
  28
  29
  30 class LoggingDecorator(subunit.test_results.HookedTestResultDecorator):
  31
  32     def __init__(self, decorated):
  33         self._calls = 0
  34         super(LoggingDecorator, self).__init__(decorated)
  35
  36     def _before_event(self):
  37         self._calls += 1
  38
  39
  40 class AssertBeforeTestResult(LoggingDecorator):
  41     """A TestResult for checking preconditions."""
  42
  43     def __init__(self, decorated, test):
  44         self.test = test
  45         super(AssertBeforeTestResult, self).__init__(decorated)
  46
  47     def _before_event(self):
  48         self.test.assertEqual(1, self.earlier._calls)
  49         super(AssertBeforeTestResult, self)._before_event()
  50
  51
  52 class TimeCapturingResult(unittest.TestResult):
  53
  54     def __init__(self):
  55         super(TimeCapturingResult, self).__init__()
  56         self._calls = []
  57
  58     def time(self, a_datetime):
  59         self._calls.append(a_datetime)
  60
  61
  62 class LoggingResult(object):
  63     """Basic support for logging of results."""
  64
  65     def __init__(self):
  66         self._calls = []
  67
  68
  69 class Python26TestResult(LoggingResult):
  70     """A python 2.6 like test result, that logs."""
  71
  72     def addError(self, test, err):
  73         self._calls.append(('addError', test, err))
  74
  75     def addFailure(self, test, err):
  76         self._calls.append(('addFailure', test, err))
  77
  78     def addSuccess(self, test):
  79         self._calls.append(('addSuccess', test))
  80
  81     def startTest(self, test):
  82         self._calls.append(('startTest', test))
  83
  84     def stopTest(self, test):
  85         self._calls.append(('stopTest', test))
  86
  87
  88 class Python27TestResult(Python26TestResult):
  89     """A python 2.7 like test result, that logs."""
  90
  91     def addExpectedFailure(self, test, err):
  92         self._calls.append(('addExpectedFailure', test, err))
  93
  94     def addSkip(self, test, reason):
  95         self._calls.append(('addSkip', test, reason))
  96
  97     def addUnexpectedSuccess(self, test):
  98         self._calls.append(('addUnexpectedSuccess', test))
  99
 100     def startTestRun(self):
 101         self._calls.append(('startTestRun',))
 102
 103     def stopTestRun(self):
 104         self._calls.append(('stopTestRun',))
 105
 106
 107 class ExtendedTestResult(Python27TestResult):
 108     """A test result like the proposed extended unittest result API."""
 109
 110     def addError(self, test, err=None, details=None):
 111         self._calls.append(('addError', test, err or details))
 112
 113     def addFailure(self, test, err=None, details=None):
 114         self._calls.append(('addFailure', test, err or details))
 115
 116     def addExpectedFailure(self, test, err=None, details=None):
 117         self._calls.append(('addExpectedFailure', test, err or details))
 118
 119     def addSkip(self, test, reason=None, details=None):
 120         self._calls.append(('addSkip', test, reason or details))
 121
 122     def addSuccess(self, test, details=None):
 123         if details:
 124             self._calls.append(('addSuccess', test, details))
 125         else:
 126             self._calls.append(('addSuccess', test))
 127
 128     def addUnexpectedSuccess(self, test, details=None):
 129         if details:
 130             self._calls.append(('addUnexpectedSuccess', test, details))
 131         else:
 132             self._calls.append(('addUnexpectedSuccess', test))
 133
 134     def progress(self, offset, whence):
 135         self._calls.append(('progress', offset, whence))
 136
 137     def tags(self, new_tags, gone_tags):
 138         self._calls.append(('tags', new_tags, gone_tags))
 139
 140
 141 class TestExtendedToOriginalResultDecoratorBase(unittest.TestCase):
 142
 143     def make_26_result(self):
 144         self.result = Python26TestResult()
 145         self.make_converter()
 146
 147     def make_27_result(self):
 148         self.result = Python27TestResult()
 149         self.make_converter()
 150
 151     def make_converter(self):
 152         self.converter = \
 153             subunit.test_results.ExtendedToOriginalDecorator(self.result)
 154
 155     def make_extended_result(self):
 156         self.result = ExtendedTestResult()
 157         self.make_converter()
 158
 159     def check_outcome_details(self, outcome):
 160         """Call an outcome with a details dict to be passed through."""
 161         # This dict is /not/ convertible - thats deliberate, as it should
 162         # not hit the conversion code path.
 163         details = {'foo': 'bar'}
 164         getattr(self.converter, outcome)(self, details=details)
 165         self.assertEqual([(outcome, self, details)], self.result._calls)
 166
 167     def get_details_and_string(self):
 168         """Get a details dict and expected string."""
 169         text1 = lambda:["1\n2\n"]
 170         text2 = lambda:["3\n4\n"]
 171         bin1 = lambda:["5\n"]
 172         details = {'text 1': Content(ContentType('text', 'plain'), text1),
 173             'text 2': Content(ContentType('text', 'strange'), text2),
 174             'bin 1': Content(ContentType('application', 'binary'), bin1)}
 175         return (details, "Binary content: bin 1\n"
 176             "Text attachment: text 1\n------------\n1\n2\n"
 177             "------------\nText attachment: text 2\n------------\n"
 178             "3\n4\n------------\n")
 179
 180     def check_outcome_details_to_exec_info(self, outcome, expected=None):
 181         """Call an outcome with a details dict to be made into exc_info."""
 182         # The conversion is a done using RemoteError and the string contents
 183         # of the text types in the details dict.
 184         if not expected:
 185             expected = outcome
 186         details, err_str = self.get_details_and_string()
 187         getattr(self.converter, outcome)(self, details=details)
 188         err = subunit.RemoteError(err_str)
 189         self.assertEqual([(expected, self, err)], self.result._calls)
 190
 191     def check_outcome_details_to_nothing(self, outcome, expected=None):
 192         """Call an outcome with a details dict to be swallowed."""
 193         if not expected:
 194             expected = outcome
 195         details = {'foo': 'bar'}
 196         getattr(self.converter, outcome)(self, details=details)
 197         self.assertEqual([(expected, self)], self.result._calls)
 198
 199     def check_outcome_details_to_string(self, outcome):
 200         """Call an outcome with a details dict to be stringified."""
 201         details, err_str = self.get_details_and_string()
 202         getattr(self.converter, outcome)(self, details=details)
 203         self.assertEqual([(outcome, self, err_str)], self.result._calls)
 204
 205     def check_outcome_exc_info(self, outcome, expected=None):
 206         """Check that calling a legacy outcome still works."""
 207         # calling some outcome with the legacy exc_info style api (no keyword
 208         # parameters) gets passed through.
 209         if not expected:
 210             expected = outcome
 211         err = subunit.RemoteError("foo\nbar\n")
 212         getattr(self.converter, outcome)(self, err)
 213         self.assertEqual([(expected, self, err)], self.result._calls)
 214
 215     def check_outcome_exc_info_to_nothing(self, outcome, expected=None):
 216         """Check that calling a legacy outcome on a fallback works."""
 217         # calling some outcome with the legacy exc_info style api (no keyword
 218         # parameters) gets passed through.
 219         if not expected:
 220             expected = outcome
 221         err = subunit.RemoteError("foo\nbar\n")
 222         getattr(self.converter, outcome)(self, err)
 223         self.assertEqual([(expected, self)], self.result._calls)
 224
 225     def check_outcome_nothing(self, outcome, expected=None):
 226         """Check that calling a legacy outcome still works."""
 227         if not expected:
 228             expected = outcome
 229         getattr(self.converter, outcome)(self)
 230         self.assertEqual([(expected, self)], self.result._calls)
 231
 232     def check_outcome_string_nothing(self, outcome, expected):
 233         """Check that calling outcome with a string calls expected."""
 234         getattr(self.converter, outcome)(self, "foo")
 235         self.assertEqual([(expected, self)], self.result._calls)
 236
 237     def check_outcome_string(self, outcome):
 238         """Check that calling outcome with a string works."""
 239         getattr(self.converter, outcome)(self, "foo")
 240         self.assertEqual([(outcome, self, "foo")], self.result._calls)
 241
 242
 243 class TestExtendedToOriginalResultDecorator(
 244     TestExtendedToOriginalResultDecoratorBase):
 245
 246     def test_progress_py26(self):
 247         self.make_26_result()
 248         self.converter.progress(1, 2)
 249
 250     def test_progress_py27(self):
 251         self.make_27_result()
 252         self.converter.progress(1, 2)
 253
 254     def test_progress_pyextended(self):
 255         self.make_extended_result()
 256         self.converter.progress(1, 2)
 257         self.assertEqual([('progress', 1, 2)], self.result._calls)
 258
 259     def test_startTest_py26(self):
 260         self.make_26_result()
 261         self.converter.startTest(self)
 262         self.assertEqual([('startTest', self)], self.result._calls)
 263
 264     def test_startTest_py27(self):
 265         self.make_27_result()
 266         self.converter.startTest(self)
 267         self.assertEqual([('startTest', self)], self.result._calls)
 268
 269     def test_startTest_pyextended(self):
 270         self.make_extended_result()
 271         self.converter.startTest(self)
 272         self.assertEqual([('startTest', self)], self.result._calls)
 273
 274     def test_startTestRun_py26(self):
 275         self.make_26_result()
 276         self.converter.startTestRun()
 277         self.assertEqual([], self.result._calls)
 278
 279     def test_startTestRun_py27(self):
 280         self.make_27_result()
 281         self.converter.startTestRun()
 282         self.assertEqual([('startTestRun',)], self.result._calls)
 283
 284     def test_startTestRun_pyextended(self):
 285         self.make_extended_result()
 286         self.converter.startTestRun()
 287         self.assertEqual([('startTestRun',)], self.result._calls)
 288
 289     def test_stopTest_py26(self):
 290         self.make_26_result()
 291         self.converter.stopTest(self)
 292         self.assertEqual([('stopTest', self)], self.result._calls)
 293
 294     def test_stopTest_py27(self):
 295         self.make_27_result()
 296         self.converter.stopTest(self)
 297         self.assertEqual([('stopTest', self)], self.result._calls)
 298
 299     def test_stopTest_pyextended(self):
 300         self.make_extended_result()
 301         self.converter.stopTest(self)
 302         self.assertEqual([('stopTest', self)], self.result._calls)
 303
 304     def test_stopTestRun_py26(self):
 305         self.make_26_result()
 306         self.converter.stopTestRun()
 307         self.assertEqual([], self.result._calls)
 308
 309     def test_stopTestRun_py27(self):
 310         self.make_27_result()
 311         self.converter.stopTestRun()
 312         self.assertEqual([('stopTestRun',)], self.result._calls)
 313
 314     def test_stopTestRun_pyextended(self):
 315         self.make_extended_result()
 316         self.converter.stopTestRun()
 317         self.assertEqual([('stopTestRun',)], self.result._calls)
 318
 319     def test_tags_py26(self):
 320         self.make_26_result()
 321         self.converter.tags(1, 2)
 322
 323     def test_tags_py27(self):
 324         self.make_27_result()
 325         self.converter.tags(1, 2)
 326
 327     def test_tags_pyextended(self):
 328         self.make_extended_result()
 329         self.converter.tags(1, 2)
 330         self.assertEqual([('tags', 1, 2)], self.result._calls)
 331
 332
 333 class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase):
 334
 335     outcome = 'addError'
 336
 337     def test_outcome_Original_py26(self):
 338         self.make_26_result()
 339         self.check_outcome_exc_info(self.outcome)
 340
 341     def test_outcome_Original_py27(self):
 342         self.make_27_result()
 343         self.check_outcome_exc_info(self.outcome)
 344
 345     def test_outcome_Original_pyextended(self):
 346         self.make_extended_result()
 347         self.check_outcome_exc_info(self.outcome)
 348
 349     def test_outcome_Extended_py26(self):
 350         self.make_26_result()
 351         self.check_outcome_details_to_exec_info(self.outcome)
 352
 353     def test_outcome_Extended_py27(self):
 354         self.make_27_result()
 355         self.check_outcome_details_to_exec_info(self.outcome)
 356
 357     def test_outcome_Extended_pyextended(self):
 358         self.make_extended_result()
 359         self.check_outcome_details(self.outcome)
 360
 361     def test_outcome__no_details(self):
 362         self.make_extended_result()
 363         self.assertRaises(ValueError,
 364             getattr(self.converter, self.outcome), self)
 365
 366
 367 class TestExtendedToOriginalAddFailure(
 368     TestExtendedToOriginalAddError):
 369
 370     outcome = 'addFailure'
 371
 372
 373 class TestExtendedToOriginalAddExpectedFailure(
 374     TestExtendedToOriginalAddError):
 375
 376     outcome = 'addExpectedFailure'
 377
 378     def test_outcome_Original_py26(self):
 379         self.make_26_result()
 380         self.check_outcome_exc_info_to_nothing(self.outcome, 'addSuccess')
 381
 382     def test_outcome_Extended_py26(self):
 383         self.make_26_result()
 384         self.check_outcome_details_to_nothing(self.outcome, 'addSuccess')
 385
 386
 387
 388 class TestExtendedToOriginalAddSkip(
 389     TestExtendedToOriginalResultDecoratorBase):
 390
 391     outcome = 'addSkip'
 392
 393     def test_outcome_Original_py26(self):
 394         self.make_26_result()
 395         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
 396
 397     def test_outcome_Original_py27(self):
 398         self.make_27_result()
 399         self.check_outcome_string(self.outcome)
 400
 401     def test_outcome_Original_pyextended(self):
 402         self.make_extended_result()
 403         self.check_outcome_string(self.outcome)
 404
 405     def test_outcome_Extended_py26(self):
 406         self.make_26_result()
 407         self.check_outcome_string_nothing(self.outcome, 'addSuccess')
 408
 409     def test_outcome_Extended_py27(self):
 410         self.make_27_result()
 411         self.check_outcome_details_to_string(self.outcome)
 412
 413     def test_outcome_Extended_pyextended(self):
 414         self.make_extended_result()
 415         self.check_outcome_details(self.outcome)
 416
 417     def test_outcome__no_details(self):
 418         self.make_extended_result()
 419         self.assertRaises(ValueError,
 420             getattr(self.converter, self.outcome), self)
 421
 422
 423 class TestExtendedToOriginalAddSuccess(
 424     TestExtendedToOriginalResultDecoratorBase):
 425
 426     outcome = 'addSuccess'
 427     expected = 'addSuccess'
 428
 429     def test_outcome_Original_py26(self):
 430         self.make_26_result()
 431         self.check_outcome_nothing(self.outcome, self.expected)
 432
 433     def test_outcome_Original_py27(self):
 434         self.make_27_result()
 435         self.check_outcome_nothing(self.outcome)
 436
 437     def test_outcome_Original_pyextended(self):
 438         self.make_extended_result()
 439         self.check_outcome_nothing(self.outcome)
 440
 441     def test_outcome_Extended_py26(self):
 442         self.make_26_result()
 443         self.check_outcome_details_to_nothing(self.outcome, self.expected)
 444
 445     def test_outcome_Extended_py27(self):
 446         self.make_27_result()
 447         self.check_outcome_details_to_nothing(self.outcome)
 448
 449     def test_outcome_Extended_pyextended(self):
 450         self.make_extended_result()
 451         self.check_outcome_details(self.outcome)
 452
 453
 454 class TestExtendedToOriginalAddUnexpectedSuccess(
 455     TestExtendedToOriginalAddSuccess):
 456
 457     outcome = 'addUnexpectedSuccess'
 458
 459
 460 class TestHookedTestResultDecorator(unittest.TestCase):
 461
 462     def setUp(self):
 463         # And end to the chain
 464         terminal = unittest.TestResult()
 465         # Asserts that the call was made to self.result before asserter was
 466         # called.
 467         asserter = AssertBeforeTestResult(terminal, self)
 468         # The result object we call, which much increase its call count.
 469         self.result = LoggingDecorator(asserter)
 470         asserter.earlier = self.result
 471
 472     def tearDown(self):
 473         # The hook in self.result must have been called
 474         self.assertEqual(1, self.result._calls)
 475         # The hook in asserter must have been called too, otherwise the
 476         # assertion about ordering won't have completed.
 477         self.assertEqual(1, self.result.decorated._calls)
 478
 479     def test_startTest(self):
 480         self.result.startTest(self)
 481
 482     def test_startTestRun(self):
 483         self.result.startTestRun()
 484
 485     def test_stopTest(self):
 486         self.result.stopTest(self)
 487
 488     def test_stopTestRun(self):
 489         self.result.stopTestRun()
 490
 491     def test_addError(self):
 492         self.result.addError(self, subunit.RemoteError())
 493
 494     def test_addFailure(self):
 495         self.result.addFailure(self, subunit.RemoteError())
 496
 497     def test_addSuccess(self):
 498         self.result.addSuccess(self)
 499
 500     def test_addSkip(self):
 501         self.result.addSkip(self, "foo")
 502
 503     def test_addExpectedFailure(self):
 504         self.result.addExpectedFailure(self, subunit.RemoteError())
 505
 506     def test_addUnexpectedSuccess(self):
 507         self.result.addUnexpectedSuccess(self)
 508
 509     def test_progress(self):
 510         self.result.progress(1, subunit.PROGRESS_SET)
 511
 512     def test_wasSuccessful(self):
 513         self.result.wasSuccessful()
 514
 515     def test_shouldStop(self):
 516         self.result.shouldStop
 517
 518     def test_stop(self):
 519         self.result.stop()
 520
 521     def test_time(self):
 522         self.result.time(None)
 523
 524
 525 class TestAutoTimingTestResultDecorator(unittest.TestCase):
 526
 527     def setUp(self):
 528         # And end to the chain which captures time events.
 529         terminal = TimeCapturingResult()
 530         # The result object under test.
 531         self.result = subunit.test_results.AutoTimingTestResultDecorator(
 532             terminal)
 533
 534     def test_without_time_calls_time_is_called_and_not_None(self):
 535         self.result.startTest(self)
 536         self.assertEqual(1, len(self.result.decorated._calls))
 537         self.assertNotEqual(None, self.result.decorated._calls[0])
 538
 539     def test_no_time_from_progress(self):
 540         self.result.progress(1, subunit.PROGRESS_CUR)
 541         self.assertEqual(0, len(self.result.decorated._calls))
 542
 543     def test_no_time_from_shouldStop(self):
 544         self.result.decorated.stop()
 545         self.result.shouldStop
 546         self.assertEqual(0, len(self.result.decorated._calls))
 547
 548     def test_calling_time_inhibits_automatic_time(self):
 549         # Calling time() outputs a time signal immediately and prevents
 550         # automatically adding one when other methods are called.
 551         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 552         self.result.time(time)
 553         self.result.startTest(self)
 554         self.result.stopTest(self)
 555         self.assertEqual(1, len(self.result.decorated._calls))
 556         self.assertEqual(time, self.result.decorated._calls[0])
 557
 558     def test_calling_time_None_enables_automatic_time(self):
 559         time = datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc())
 560         self.result.time(time)
 561         self.assertEqual(1, len(self.result.decorated._calls))
 562         self.assertEqual(time, self.result.decorated._calls[0])
 563         # Calling None passes the None through, in case other results care.
 564         self.result.time(None)
 565         self.assertEqual(2, len(self.result.decorated._calls))
 566         self.assertEqual(None, self.result.decorated._calls[1])
 567         # Calling other methods doesn't generate an automatic time event.
 568         self.result.startTest(self)
 569         self.assertEqual(3, len(self.result.decorated._calls))
 570         self.assertNotEqual(None, self.result.decorated._calls[2])
 571
 572
 573 def test_suite():
 574     loader = subunit.tests.TestUtil.TestLoader()
 575     result = loader.loadTestsFromName(__name__)
 576     return result