git.samba.org - third_party/subunit/blob - python/subunit/tests/test_test_protocol.py

   1 #
   2 #  subunit: extensions to Python unittest to get test results from subprocesses.
   3 #  Copyright (C) 2005  Robert Collins <robertc@robertcollins.net>
   4 #
   5 #  This program is free software; you can redistribute it and/or modify
   6 #  it under the terms of the GNU General Public License as published by
   7 #  the Free Software Foundation; either version 2 of the License, or
   8 #  (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18 #
  19
  20 import datetime
  21 import unittest
  22 from StringIO import StringIO
  23 import os
  24 import subunit
  25 import sys
  26
  27 import subunit.iso8601 as iso8601
  28
  29
  30 class MockTestProtocolServerClient(object):
  31     """A mock protocol server client to test callbacks."""
  32
  33     def __init__(self):
  34         self.end_calls = []
  35         self.error_calls = []
  36         self.failure_calls = []
  37         self.skip_calls = []
  38         self.start_calls = []
  39         self.success_calls = []
  40         self._time = None
  41         super(MockTestProtocolServerClient, self).__init__()
  42
  43     def addError(self, test, error):
  44         self.error_calls.append((test, error))
  45
  46     def addFailure(self, test, error):
  47         self.failure_calls.append((test, error))
  48
  49     def addSkip(self, test, reason):
  50         self.skip_calls.append((test, reason))
  51
  52     def addSuccess(self, test):
  53         self.success_calls.append(test)
  54
  55     def stopTest(self, test):
  56         self.end_calls.append(test)
  57
  58     def startTest(self, test):
  59         self.start_calls.append(test)
  60
  61     def time(self, time):
  62         self._time = time
  63
  64
  65 class TestMockTestProtocolServer(unittest.TestCase):
  66
  67     def test_start_test(self):
  68         protocol = MockTestProtocolServerClient()
  69         protocol.startTest(subunit.RemotedTestCase("test old mcdonald"))
  70         self.assertEqual(protocol.start_calls,
  71                          [subunit.RemotedTestCase("test old mcdonald")])
  72         self.assertEqual(protocol.end_calls, [])
  73         self.assertEqual(protocol.error_calls, [])
  74         self.assertEqual(protocol.failure_calls, [])
  75         self.assertEqual(protocol.success_calls, [])
  76
  77     def test_add_error(self):
  78         protocol = MockTestProtocolServerClient()
  79         protocol.addError(subunit.RemotedTestCase("old mcdonald"),
  80                           subunit.RemoteError("omg it works"))
  81         self.assertEqual(protocol.start_calls, [])
  82         self.assertEqual(protocol.end_calls, [])
  83         self.assertEqual(protocol.error_calls, [(
  84                             subunit.RemotedTestCase("old mcdonald"),
  85                             subunit.RemoteError("omg it works"))])
  86         self.assertEqual(protocol.failure_calls, [])
  87         self.assertEqual(protocol.success_calls, [])
  88
  89     def test_add_failure(self):
  90         protocol = MockTestProtocolServerClient()
  91         protocol.addFailure(subunit.RemotedTestCase("old mcdonald"),
  92                             subunit.RemoteError("omg it works"))
  93         self.assertEqual(protocol.start_calls, [])
  94         self.assertEqual(protocol.end_calls, [])
  95         self.assertEqual(protocol.error_calls, [])
  96         self.assertEqual(protocol.failure_calls, [
  97                             (subunit.RemotedTestCase("old mcdonald"),
  98                              subunit.RemoteError("omg it works"))])
  99         self.assertEqual(protocol.success_calls, [])
 100
 101     def test_add_success(self):
 102         protocol = MockTestProtocolServerClient()
 103         protocol.addSuccess(subunit.RemotedTestCase("test old mcdonald"))
 104         self.assertEqual(protocol.start_calls, [])
 105         self.assertEqual(protocol.end_calls, [])
 106         self.assertEqual(protocol.error_calls, [])
 107         self.assertEqual(protocol.failure_calls, [])
 108         self.assertEqual(protocol.success_calls,
 109                          [subunit.RemotedTestCase("test old mcdonald")])
 110
 111     def test_end_test(self):
 112         protocol = MockTestProtocolServerClient()
 113         protocol.stopTest(subunit.RemotedTestCase("test old mcdonald"))
 114         self.assertEqual(protocol.end_calls,
 115                          [subunit.RemotedTestCase("test old mcdonald")])
 116         self.assertEqual(protocol.error_calls, [])
 117         self.assertEqual(protocol.failure_calls, [])
 118         self.assertEqual(protocol.success_calls, [])
 119         self.assertEqual(protocol.start_calls, [])
 120
 121
 122 class TestTestImports(unittest.TestCase):
 123
 124     def test_imports(self):
 125         from subunit import TestProtocolServer
 126         from subunit import RemotedTestCase
 127         from subunit import RemoteError
 128         from subunit import ExecTestCase
 129         from subunit import IsolatedTestCase
 130         from subunit import TestProtocolClient
 131
 132
 133 class TestTestProtocolServerPipe(unittest.TestCase):
 134
 135     def test_story(self):
 136         client = unittest.TestResult()
 137         protocol = subunit.TestProtocolServer(client)
 138         pipe = StringIO("test old mcdonald\n"
 139                         "success old mcdonald\n"
 140                         "test bing crosby\n"
 141                         "failure bing crosby [\n"
 142                         "foo.c:53:ERROR invalid state\n"
 143                         "]\n"
 144                         "test an error\n"
 145                         "error an error\n")
 146         protocol.readFrom(pipe)
 147         mcdonald = subunit.RemotedTestCase("old mcdonald")
 148         bing = subunit.RemotedTestCase("bing crosby")
 149         an_error = subunit.RemotedTestCase("an error")
 150         self.assertEqual(client.errors,
 151                          [(an_error, 'RemoteException: \n\n')])
 152         self.assertEqual(
 153             client.failures,
 154             [(bing, "RemoteException: foo.c:53:ERROR invalid state\n\n")])
 155         self.assertEqual(client.testsRun, 3)
 156
 157
 158 class TestTestProtocolServerStartTest(unittest.TestCase):
 159
 160     def setUp(self):
 161         self.client = MockTestProtocolServerClient()
 162         self.protocol = subunit.TestProtocolServer(self.client)
 163
 164     def test_start_test(self):
 165         self.protocol.lineReceived("test old mcdonald\n")
 166         self.assertEqual(self.client.start_calls,
 167                          [subunit.RemotedTestCase("old mcdonald")])
 168
 169     def test_start_testing(self):
 170         self.protocol.lineReceived("testing old mcdonald\n")
 171         self.assertEqual(self.client.start_calls,
 172                          [subunit.RemotedTestCase("old mcdonald")])
 173
 174     def test_start_test_colon(self):
 175         self.protocol.lineReceived("test: old mcdonald\n")
 176         self.assertEqual(self.client.start_calls,
 177                          [subunit.RemotedTestCase("old mcdonald")])
 178
 179     def test_start_testing_colon(self):
 180         self.protocol.lineReceived("testing: old mcdonald\n")
 181         self.assertEqual(self.client.start_calls,
 182                          [subunit.RemotedTestCase("old mcdonald")])
 183
 184
 185 class TestTestProtocolServerPassThrough(unittest.TestCase):
 186
 187     def setUp(self):
 188         from StringIO import StringIO
 189         self.stdout = StringIO()
 190         self.test = subunit.RemotedTestCase("old mcdonald")
 191         self.client = MockTestProtocolServerClient()
 192         self.protocol = subunit.TestProtocolServer(self.client, self.stdout)
 193
 194     def keywords_before_test(self):
 195         self.protocol.lineReceived("failure a\n")
 196         self.protocol.lineReceived("failure: a\n")
 197         self.protocol.lineReceived("error a\n")
 198         self.protocol.lineReceived("error: a\n")
 199         self.protocol.lineReceived("success a\n")
 200         self.protocol.lineReceived("success: a\n")
 201         self.protocol.lineReceived("successful a\n")
 202         self.protocol.lineReceived("successful: a\n")
 203         self.protocol.lineReceived("]\n")
 204         self.assertEqual(self.stdout.getvalue(), "failure a\n"
 205                                                  "failure: a\n"
 206                                                  "error a\n"
 207                                                  "error: a\n"
 208                                                  "success a\n"
 209                                                  "success: a\n"
 210                                                  "successful a\n"
 211                                                  "successful: a\n"
 212                                                  "]\n")
 213
 214     def test_keywords_before_test(self):
 215         self.keywords_before_test()
 216         self.assertEqual(self.client.start_calls, [])
 217         self.assertEqual(self.client.error_calls, [])
 218         self.assertEqual(self.client.failure_calls, [])
 219         self.assertEqual(self.client.success_calls, [])
 220
 221     def test_keywords_after_error(self):
 222         self.protocol.lineReceived("test old mcdonald\n")
 223         self.protocol.lineReceived("error old mcdonald\n")
 224         self.keywords_before_test()
 225         self.assertEqual(self.client.start_calls, [self.test])
 226         self.assertEqual(self.client.end_calls, [self.test])
 227         self.assertEqual(self.client.error_calls,
 228                          [(self.test, subunit.RemoteError(""))])
 229         self.assertEqual(self.client.failure_calls, [])
 230         self.assertEqual(self.client.success_calls, [])
 231
 232     def test_keywords_after_failure(self):
 233         self.protocol.lineReceived("test old mcdonald\n")
 234         self.protocol.lineReceived("failure old mcdonald\n")
 235         self.keywords_before_test()
 236         self.assertEqual(self.client.start_calls, [self.test])
 237         self.assertEqual(self.client.end_calls, [self.test])
 238         self.assertEqual(self.client.error_calls, [])
 239         self.assertEqual(self.client.failure_calls,
 240                          [(self.test, subunit.RemoteError())])
 241         self.assertEqual(self.client.success_calls, [])
 242
 243     def test_keywords_after_success(self):
 244         self.protocol.lineReceived("test old mcdonald\n")
 245         self.protocol.lineReceived("success old mcdonald\n")
 246         self.keywords_before_test()
 247         self.assertEqual(self.client.start_calls, [self.test])
 248         self.assertEqual(self.client.end_calls, [self.test])
 249         self.assertEqual(self.client.error_calls, [])
 250         self.assertEqual(self.client.failure_calls, [])
 251         self.assertEqual(self.client.success_calls, [self.test])
 252
 253     def test_keywords_after_test(self):
 254         self.protocol.lineReceived("test old mcdonald\n")
 255         self.protocol.lineReceived("test old mcdonald\n")
 256         self.protocol.lineReceived("failure a\n")
 257         self.protocol.lineReceived("failure: a\n")
 258         self.protocol.lineReceived("error a\n")
 259         self.protocol.lineReceived("error: a\n")
 260         self.protocol.lineReceived("success a\n")
 261         self.protocol.lineReceived("success: a\n")
 262         self.protocol.lineReceived("successful a\n")
 263         self.protocol.lineReceived("successful: a\n")
 264         self.protocol.lineReceived("]\n")
 265         self.protocol.lineReceived("failure old mcdonald\n")
 266         self.assertEqual(self.stdout.getvalue(), "test old mcdonald\n"
 267                                                  "failure a\n"
 268                                                  "failure: a\n"
 269                                                  "error a\n"
 270                                                  "error: a\n"
 271                                                  "success a\n"
 272                                                  "success: a\n"
 273                                                  "successful a\n"
 274                                                  "successful: a\n"
 275                                                  "]\n")
 276         self.assertEqual(self.client.start_calls, [self.test])
 277         self.assertEqual(self.client.end_calls, [self.test])
 278         self.assertEqual(self.client.failure_calls,
 279                          [(self.test, subunit.RemoteError())])
 280         self.assertEqual(self.client.error_calls, [])
 281         self.assertEqual(self.client.success_calls, [])
 282
 283     def test_keywords_during_failure(self):
 284         self.protocol.lineReceived("test old mcdonald\n")
 285         self.protocol.lineReceived("failure: old mcdonald [\n")
 286         self.protocol.lineReceived("test old mcdonald\n")
 287         self.protocol.lineReceived("failure a\n")
 288         self.protocol.lineReceived("failure: a\n")
 289         self.protocol.lineReceived("error a\n")
 290         self.protocol.lineReceived("error: a\n")
 291         self.protocol.lineReceived("success a\n")
 292         self.protocol.lineReceived("success: a\n")
 293         self.protocol.lineReceived("successful a\n")
 294         self.protocol.lineReceived("successful: a\n")
 295         self.protocol.lineReceived(" ]\n")
 296         self.protocol.lineReceived("]\n")
 297         self.assertEqual(self.stdout.getvalue(), "")
 298         self.assertEqual(self.client.start_calls, [self.test])
 299         self.assertEqual(self.client.failure_calls,
 300                          [(self.test, subunit.RemoteError("test old mcdonald\n"
 301                                                   "failure a\n"
 302                                                   "failure: a\n"
 303                                                   "error a\n"
 304                                                   "error: a\n"
 305                                                   "success a\n"
 306                                                   "success: a\n"
 307                                                   "successful a\n"
 308                                                   "successful: a\n"
 309                                                   "]\n"))])
 310         self.assertEqual(self.client.end_calls, [self.test])
 311         self.assertEqual(self.client.error_calls, [])
 312         self.assertEqual(self.client.success_calls, [])
 313
 314     def test_stdout_passthrough(self):
 315         """Lines received which cannot be interpreted as any protocol action
 316         should be passed through to sys.stdout.
 317         """
 318         bytes = "randombytes\n"
 319         self.protocol.lineReceived(bytes)
 320         self.assertEqual(self.stdout.getvalue(), bytes)
 321
 322
 323 class TestTestProtocolServerLostConnection(unittest.TestCase):
 324
 325     def setUp(self):
 326         self.client = MockTestProtocolServerClient()
 327         self.protocol = subunit.TestProtocolServer(self.client)
 328         self.test = subunit.RemotedTestCase("old mcdonald")
 329
 330     def test_lost_connection_no_input(self):
 331         self.protocol.lostConnection()
 332         self.assertEqual(self.client.start_calls, [])
 333         self.assertEqual(self.client.error_calls, [])
 334         self.assertEqual(self.client.failure_calls, [])
 335         self.assertEqual(self.client.success_calls, [])
 336
 337     def test_lost_connection_after_start(self):
 338         self.protocol.lineReceived("test old mcdonald\n")
 339         self.protocol.lostConnection()
 340         self.assertEqual(self.client.start_calls, [self.test])
 341         self.assertEqual(self.client.end_calls, [self.test])
 342         self.assertEqual(self.client.error_calls, [
 343             (self.test, subunit.RemoteError("lost connection during "
 344                                             "test 'old mcdonald'"))])
 345         self.assertEqual(self.client.failure_calls, [])
 346         self.assertEqual(self.client.success_calls, [])
 347
 348     def test_lost_connected_after_error(self):
 349         self.protocol.lineReceived("test old mcdonald\n")
 350         self.protocol.lineReceived("error old mcdonald\n")
 351         self.protocol.lostConnection()
 352         self.assertEqual(self.client.start_calls, [self.test])
 353         self.assertEqual(self.client.failure_calls, [])
 354         self.assertEqual(self.client.end_calls, [self.test])
 355         self.assertEqual(self.client.error_calls, [
 356             (self.test, subunit.RemoteError(""))])
 357         self.assertEqual(self.client.success_calls, [])
 358
 359     def test_lost_connection_during_error(self):
 360         self.protocol.lineReceived("test old mcdonald\n")
 361         self.protocol.lineReceived("error old mcdonald [\n")
 362         self.protocol.lostConnection()
 363         self.assertEqual(self.client.start_calls, [self.test])
 364         self.assertEqual(self.client.end_calls, [self.test])
 365         self.assertEqual(self.client.error_calls, [
 366             (self.test, subunit.RemoteError("lost connection during error "
 367                                             "report of test 'old mcdonald'"))])
 368         self.assertEqual(self.client.failure_calls, [])
 369         self.assertEqual(self.client.success_calls, [])
 370
 371     def test_lost_connected_after_failure(self):
 372         self.protocol.lineReceived("test old mcdonald\n")
 373         self.protocol.lineReceived("failure old mcdonald\n")
 374         self.protocol.lostConnection()
 375         test = subunit.RemotedTestCase("old mcdonald")
 376         self.assertEqual(self.client.start_calls, [self.test])
 377         self.assertEqual(self.client.end_calls, [self.test])
 378         self.assertEqual(self.client.error_calls, [])
 379         self.assertEqual(self.client.failure_calls,
 380                          [(self.test, subunit.RemoteError())])
 381         self.assertEqual(self.client.success_calls, [])
 382
 383     def test_lost_connection_during_failure(self):
 384         self.protocol.lineReceived("test old mcdonald\n")
 385         self.protocol.lineReceived("failure old mcdonald [\n")
 386         self.protocol.lostConnection()
 387         self.assertEqual(self.client.start_calls, [self.test])
 388         self.assertEqual(self.client.end_calls, [self.test])
 389         self.assertEqual(self.client.error_calls,
 390                          [(self.test,
 391                            subunit.RemoteError("lost connection during "
 392                                                "failure report"
 393                                                " of test 'old mcdonald'"))])
 394         self.assertEqual(self.client.failure_calls, [])
 395         self.assertEqual(self.client.success_calls, [])
 396
 397     def test_lost_connection_after_success(self):
 398         self.protocol.lineReceived("test old mcdonald\n")
 399         self.protocol.lineReceived("success old mcdonald\n")
 400         self.protocol.lostConnection()
 401         self.assertEqual(self.client.start_calls, [self.test])
 402         self.assertEqual(self.client.end_calls, [self.test])
 403         self.assertEqual(self.client.error_calls, [])
 404         self.assertEqual(self.client.failure_calls, [])
 405         self.assertEqual(self.client.success_calls, [self.test])
 406
 407     def test_lost_connection_during_skip(self):
 408         self.protocol.lineReceived("test old mcdonald\n")
 409         self.protocol.lineReceived("skip old mcdonald [\n")
 410         self.protocol.lostConnection()
 411         self.assertEqual(self.client.start_calls, [self.test])
 412         self.assertEqual(self.client.end_calls, [self.test])
 413         self.assertEqual(self.client.error_calls, [
 414             (self.test, subunit.RemoteError("lost connection during skip "
 415                                             "report of test 'old mcdonald'"))])
 416         self.assertEqual(self.client.failure_calls, [])
 417         self.assertEqual(self.client.success_calls, [])
 418
 419     def test_lost_connection_during_xfail(self):
 420         self.protocol.lineReceived("test old mcdonald\n")
 421         self.protocol.lineReceived("xfail old mcdonald [\n")
 422         self.protocol.lostConnection()
 423         self.assertEqual(self.client.start_calls, [self.test])
 424         self.assertEqual(self.client.end_calls, [self.test])
 425         self.assertEqual(self.client.error_calls, [
 426             (self.test, subunit.RemoteError("lost connection during xfail "
 427                                             "report of test 'old mcdonald'"))])
 428         self.assertEqual(self.client.failure_calls, [])
 429         self.assertEqual(self.client.success_calls, [])
 430
 431     def test_lost_connection_during_success(self):
 432         self.protocol.lineReceived("test old mcdonald\n")
 433         self.protocol.lineReceived("success old mcdonald [\n")
 434         self.protocol.lostConnection()
 435         self.assertEqual(self.client.start_calls, [self.test])
 436         self.assertEqual(self.client.end_calls, [self.test])
 437         self.assertEqual(self.client.error_calls, [
 438             (self.test, subunit.RemoteError("lost connection during success "
 439                                             "report of test 'old mcdonald'"))])
 440         self.assertEqual(self.client.failure_calls, [])
 441         self.assertEqual(self.client.success_calls, [])
 442
 443
 444 class TestTestProtocolServerAddError(unittest.TestCase):
 445
 446     def setUp(self):
 447         self.client = MockTestProtocolServerClient()
 448         self.protocol = subunit.TestProtocolServer(self.client)
 449         self.protocol.lineReceived("test mcdonalds farm\n")
 450         self.test = subunit.RemotedTestCase("mcdonalds farm")
 451
 452     def simple_error_keyword(self, keyword):
 453         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
 454         self.assertEqual(self.client.start_calls, [self.test])
 455         self.assertEqual(self.client.end_calls, [self.test])
 456         self.assertEqual(self.client.error_calls, [
 457             (self.test, subunit.RemoteError(""))])
 458         self.assertEqual(self.client.failure_calls, [])
 459
 460     def test_simple_error(self):
 461         self.simple_error_keyword("error")
 462
 463     def test_simple_error_colon(self):
 464         self.simple_error_keyword("error:")
 465
 466     def test_error_empty_message(self):
 467         self.protocol.lineReceived("error mcdonalds farm [\n")
 468         self.protocol.lineReceived("]\n")
 469         self.assertEqual(self.client.start_calls, [self.test])
 470         self.assertEqual(self.client.end_calls, [self.test])
 471         self.assertEqual(self.client.error_calls, [
 472             (self.test, subunit.RemoteError(""))])
 473         self.assertEqual(self.client.failure_calls, [])
 474
 475     def error_quoted_bracket(self, keyword):
 476         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
 477         self.protocol.lineReceived(" ]\n")
 478         self.protocol.lineReceived("]\n")
 479         self.assertEqual(self.client.start_calls, [self.test])
 480         self.assertEqual(self.client.end_calls, [self.test])
 481         self.assertEqual(self.client.error_calls, [
 482             (self.test, subunit.RemoteError("]\n"))])
 483         self.assertEqual(self.client.failure_calls, [])
 484
 485     def test_error_quoted_bracket(self):
 486         self.error_quoted_bracket("error")
 487
 488     def test_error_colon_quoted_bracket(self):
 489         self.error_quoted_bracket("error:")
 490
 491
 492 class TestTestProtocolServerAddFailure(unittest.TestCase):
 493
 494     def setUp(self):
 495         self.client = MockTestProtocolServerClient()
 496         self.protocol = subunit.TestProtocolServer(self.client)
 497         self.protocol.lineReceived("test mcdonalds farm\n")
 498         self.test = subunit.RemotedTestCase("mcdonalds farm")
 499
 500     def simple_failure_keyword(self, keyword):
 501         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
 502         self.assertEqual(self.client.start_calls, [self.test])
 503         self.assertEqual(self.client.end_calls, [self.test])
 504         self.assertEqual(self.client.error_calls, [])
 505         self.assertEqual(self.client.failure_calls,
 506                          [(self.test, subunit.RemoteError())])
 507
 508     def test_simple_failure(self):
 509         self.simple_failure_keyword("failure")
 510
 511     def test_simple_failure_colon(self):
 512         self.simple_failure_keyword("failure:")
 513
 514     def test_failure_empty_message(self):
 515         self.protocol.lineReceived("failure mcdonalds farm [\n")
 516         self.protocol.lineReceived("]\n")
 517         self.assertEqual(self.client.start_calls, [self.test])
 518         self.assertEqual(self.client.end_calls, [self.test])
 519         self.assertEqual(self.client.error_calls, [])
 520         self.assertEqual(self.client.failure_calls,
 521                          [(self.test, subunit.RemoteError())])
 522
 523     def failure_quoted_bracket(self, keyword):
 524         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
 525         self.protocol.lineReceived(" ]\n")
 526         self.protocol.lineReceived("]\n")
 527         self.assertEqual(self.client.start_calls, [self.test])
 528         self.assertEqual(self.client.end_calls, [self.test])
 529         self.assertEqual(self.client.error_calls, [])
 530         self.assertEqual(self.client.failure_calls,
 531                          [(self.test, subunit.RemoteError("]\n"))])
 532
 533     def test_failure_quoted_bracket(self):
 534         self.failure_quoted_bracket("failure")
 535
 536     def test_failure_colon_quoted_bracket(self):
 537         self.failure_quoted_bracket("failure:")
 538
 539
 540 class TestTestProtocolServerAddxFail(unittest.TestCase):
 541     """Tests for the xfail keyword.
 542
 543     In Python this thunks through to Success due to stdlib limitations (see
 544     README).
 545     """
 546
 547     def setUp(self):
 548         """Setup a test object ready to be xfailed."""
 549         self.client = MockTestProtocolServerClient()
 550         self.protocol = subunit.TestProtocolServer(self.client)
 551         self.protocol.lineReceived("test mcdonalds farm\n")
 552         self.test = self.client.start_calls[-1]
 553
 554     def simple_xfail_keyword(self, keyword):
 555         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
 556         self.assertEqual(self.client.start_calls, [self.test])
 557         self.assertEqual(self.client.end_calls, [self.test])
 558         self.assertEqual(self.client.error_calls, [])
 559         self.assertEqual(self.client.failure_calls, [])
 560         self.assertEqual(self.client.success_calls, [self.test])
 561
 562     def test_simple_xfail(self):
 563         self.simple_xfail_keyword("xfail")
 564
 565     def test_simple_xfail_colon(self):
 566         self.simple_xfail_keyword("xfail:")
 567
 568     def test_xfail_empty_message(self):
 569         self.protocol.lineReceived("xfail mcdonalds farm [\n")
 570         self.protocol.lineReceived("]\n")
 571         self.assertEqual(self.client.start_calls, [self.test])
 572         self.assertEqual(self.client.end_calls, [self.test])
 573         self.assertEqual(self.client.error_calls, [])
 574         self.assertEqual(self.client.failure_calls, [])
 575         self.assertEqual(self.client.success_calls, [self.test])
 576
 577     def xfail_quoted_bracket(self, keyword):
 578         # This tests it is accepted, but cannot test it is used today, because
 579         # of not having a way to expose it in Python so far.
 580         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
 581         self.protocol.lineReceived(" ]\n")
 582         self.protocol.lineReceived("]\n")
 583         self.assertEqual(self.client.start_calls, [self.test])
 584         self.assertEqual(self.client.end_calls, [self.test])
 585         self.assertEqual(self.client.error_calls, [])
 586         self.assertEqual(self.client.failure_calls, [])
 587         self.assertEqual(self.client.success_calls, [self.test])
 588
 589     def test_xfail_quoted_bracket(self):
 590         self.xfail_quoted_bracket("xfail")
 591
 592     def test_xfail_colon_quoted_bracket(self):
 593         self.xfail_quoted_bracket("xfail:")
 594
 595
 596 class TestTestProtocolServerAddSkip(unittest.TestCase):
 597     """Tests for the skip keyword.
 598
 599     In python this meets the testtools extended TestResult contract.
 600     (See https://launchpad.net/testtools).
 601     """
 602
 603     def setUp(self):
 604         """Setup a test object ready to be skipped."""
 605         self.client = MockTestProtocolServerClient()
 606         self.protocol = subunit.TestProtocolServer(self.client)
 607         self.protocol.lineReceived("test mcdonalds farm\n")
 608         self.test = self.client.start_calls[-1]
 609
 610     def simple_skip_keyword(self, keyword):
 611         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
 612         self.assertEqual(self.client.start_calls, [self.test])
 613         self.assertEqual(self.client.end_calls, [self.test])
 614         self.assertEqual(self.client.error_calls, [])
 615         self.assertEqual(self.client.failure_calls, [])
 616         self.assertEqual(self.client.success_calls, [])
 617         self.assertEqual(self.client.skip_calls,
 618             [(self.test, 'No reason given')])
 619
 620     def test_simple_skip(self):
 621         self.simple_skip_keyword("skip")
 622
 623     def test_simple_skip_colon(self):
 624         self.simple_skip_keyword("skip:")
 625
 626     def test_skip_empty_message(self):
 627         self.protocol.lineReceived("skip mcdonalds farm [\n")
 628         self.protocol.lineReceived("]\n")
 629         self.assertEqual(self.client.start_calls, [self.test])
 630         self.assertEqual(self.client.end_calls, [self.test])
 631         self.assertEqual(self.client.error_calls, [])
 632         self.assertEqual(self.client.failure_calls, [])
 633         self.assertEqual(self.client.success_calls, [])
 634         self.assertEqual(self.client.skip_calls,
 635             [(self.test, "No reason given")])
 636
 637     def skip_quoted_bracket(self, keyword):
 638         # This tests it is accepted, but cannot test it is used today, because
 639         # of not having a way to expose it in Python so far.
 640         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
 641         self.protocol.lineReceived(" ]\n")
 642         self.protocol.lineReceived("]\n")
 643         self.assertEqual(self.client.start_calls, [self.test])
 644         self.assertEqual(self.client.end_calls, [self.test])
 645         self.assertEqual(self.client.error_calls, [])
 646         self.assertEqual(self.client.failure_calls, [])
 647         self.assertEqual(self.client.success_calls, [])
 648         self.assertEqual(self.client.skip_calls,
 649             [(self.test, "]\n")])
 650
 651     def test_skip_quoted_bracket(self):
 652         self.skip_quoted_bracket("skip")
 653
 654     def test_skip_colon_quoted_bracket(self):
 655         self.skip_quoted_bracket("skip:")
 656
 657
 658 class TestTestProtocolServerAddSuccess(unittest.TestCase):
 659
 660     def setUp(self):
 661         self.client = MockTestProtocolServerClient()
 662         self.protocol = subunit.TestProtocolServer(self.client)
 663         self.protocol.lineReceived("test mcdonalds farm\n")
 664         self.test = subunit.RemotedTestCase("mcdonalds farm")
 665
 666     def simple_success_keyword(self, keyword):
 667         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
 668         self.assertEqual(self.client.start_calls, [self.test])
 669         self.assertEqual(self.client.end_calls, [self.test])
 670         self.assertEqual(self.client.error_calls, [])
 671         self.assertEqual(self.client.success_calls, [self.test])
 672
 673     def test_simple_success(self):
 674         self.simple_success_keyword("failure")
 675
 676     def test_simple_success_colon(self):
 677         self.simple_success_keyword("failure:")
 678
 679     def test_simple_success(self):
 680         self.simple_success_keyword("successful")
 681
 682     def test_simple_success_colon(self):
 683         self.simple_success_keyword("successful:")
 684
 685     def test_success_empty_message(self):
 686         self.protocol.lineReceived("success mcdonalds farm [\n")
 687         self.protocol.lineReceived("]\n")
 688         self.assertEqual(self.client.start_calls, [self.test])
 689         self.assertEqual(self.client.end_calls, [self.test])
 690         self.assertEqual(self.client.error_calls, [])
 691         self.assertEqual(self.client.failure_calls, [])
 692         self.assertEqual(self.client.success_calls, [self.test])
 693
 694     def success_quoted_bracket(self, keyword):
 695         # This tests it is accepted, but cannot test it is used today, because
 696         # of not having a way to expose it in Python so far.
 697         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
 698         self.protocol.lineReceived(" ]\n")
 699         self.protocol.lineReceived("]\n")
 700         self.assertEqual(self.client.start_calls, [self.test])
 701         self.assertEqual(self.client.end_calls, [self.test])
 702         self.assertEqual(self.client.error_calls, [])
 703         self.assertEqual(self.client.failure_calls, [])
 704         self.assertEqual(self.client.success_calls, [self.test])
 705
 706     def test_success_quoted_bracket(self):
 707         self.success_quoted_bracket("success")
 708
 709     def test_success_colon_quoted_bracket(self):
 710         self.success_quoted_bracket("success:")
 711
 712
 713 class TestTestProtocolServerStreamTags(unittest.TestCase):
 714     """Test managing tags on the protocol level."""
 715
 716     def setUp(self):
 717         self.client = MockTestProtocolServerClient()
 718         self.protocol = subunit.TestProtocolServer(self.client)
 719
 720     def test_initial_tags(self):
 721         self.protocol.lineReceived("tags: foo bar:baz  quux\n")
 722         self.assertEqual(set(["foo", "bar:baz", "quux"]),
 723             self.protocol.tags)
 724
 725     def test_minus_removes_tags(self):
 726         self.protocol.lineReceived("tags: foo bar\n")
 727         self.protocol.lineReceived("tags: -bar quux\n")
 728         self.assertEqual(set(["foo", "quux"]),
 729             self.protocol.tags)
 730
 731     def test_tags_get_set_on_test_no_tags(self):
 732         self.protocol.lineReceived("test mcdonalds farm\n")
 733         test = self.client.start_calls[-1]
 734         self.assertEqual(set(), test.tags)
 735
 736     def test_tags_get_set_on_test_protocol_tags_only(self):
 737         self.protocol.lineReceived("tags: foo bar\n")
 738         self.protocol.lineReceived("test mcdonalds farm\n")
 739         test = self.client.start_calls[-1]
 740         self.assertEqual(set(["foo", "bar"]), test.tags)
 741
 742     def test_tags_get_set_on_test_simple(self):
 743         self.protocol.lineReceived("test mcdonalds farm\n")
 744         test = self.client.start_calls[-1]
 745         self.protocol.lineReceived("tags: foo bar\n")
 746         self.assertEqual(set(["foo", "bar"]), test.tags)
 747         self.assertEqual(set(), self.protocol.tags)
 748
 749     def test_tags_get_set_on_test_minus_removes(self):
 750         self.protocol.lineReceived("test mcdonalds farm\n")
 751         test = self.client.start_calls[-1]
 752         self.protocol.lineReceived("tags: foo bar\n")
 753         self.protocol.lineReceived("tags: -bar quux\n")
 754         self.assertEqual(set(["foo", "quux"]), test.tags)
 755         self.assertEqual(set(), self.protocol.tags)
 756
 757     def test_test_tags_inherit_protocol_tags(self):
 758         self.protocol.lineReceived("tags: foo bar\n")
 759         self.protocol.lineReceived("test mcdonalds farm\n")
 760         test = self.client.start_calls[-1]
 761         self.protocol.lineReceived("tags: -bar quux\n")
 762         self.assertEqual(set(["foo", "quux"]), test.tags)
 763         self.assertEqual(set(["foo", "bar"]), self.protocol.tags)
 764
 765
 766 class TestTestProtocolServerStreamTime(unittest.TestCase):
 767     """Test managing time information at the protocol level."""
 768
 769     def test_time_accepted_stdlib(self):
 770         self.result = unittest.TestResult()
 771         self.stream = StringIO()
 772         self.protocol = subunit.TestProtocolServer(self.result,
 773             stream=self.stream)
 774         self.protocol.lineReceived("time: 2001-12-12 12:59:59Z\n")
 775         self.assertEqual("", self.stream.getvalue())
 776
 777     def test_time_accepted_extended(self):
 778         self.result = MockTestProtocolServerClient()
 779         self.stream = StringIO()
 780         self.protocol = subunit.TestProtocolServer(self.result,
 781             stream=self.stream)
 782         self.protocol.lineReceived("time: 2001-12-12 12:59:59Z\n")
 783         self.assertEqual("", self.stream.getvalue())
 784         self.assertEqual(datetime.datetime(2001, 12, 12, 12, 59, 59, 0,
 785             iso8601.Utc()), self.result._time)
 786
 787
 788 class TestRemotedTestCase(unittest.TestCase):
 789
 790     def test_simple(self):
 791         test = subunit.RemotedTestCase("A test description")
 792         self.assertRaises(NotImplementedError, test.setUp)
 793         self.assertRaises(NotImplementedError, test.tearDown)
 794         self.assertEqual("A test description",
 795                          test.shortDescription())
 796         self.assertEqual("A test description",
 797                          test.id())
 798         self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
 799         self.assertEqual("<subunit.RemotedTestCase description="
 800                          "'A test description'>", "%r" % test)
 801         result = unittest.TestResult()
 802         test.run(result)
 803         self.assertEqual([(test, "RemoteException: "
 804                                  "Cannot run RemotedTestCases.\n\n")],
 805                          result.errors)
 806         self.assertEqual(1, result.testsRun)
 807         another_test = subunit.RemotedTestCase("A test description")
 808         self.assertEqual(test, another_test)
 809         different_test = subunit.RemotedTestCase("ofo")
 810         self.assertNotEqual(test, different_test)
 811         self.assertNotEqual(another_test, different_test)
 812
 813
 814 class TestRemoteError(unittest.TestCase):
 815
 816     def test_eq(self):
 817         error = subunit.RemoteError("Something went wrong")
 818         another_error = subunit.RemoteError("Something went wrong")
 819         different_error = subunit.RemoteError("boo!")
 820         self.assertEqual(error, another_error)
 821         self.assertNotEqual(error, different_error)
 822         self.assertNotEqual(different_error, another_error)
 823
 824     def test_empty_constructor(self):
 825         self.assertEqual(subunit.RemoteError(), subunit.RemoteError(""))
 826
 827
 828 class TestExecTestCase(unittest.TestCase):
 829
 830     class SampleExecTestCase(subunit.ExecTestCase):
 831
 832         def test_sample_method(self):
 833             """sample-script.py"""
 834             # the sample script runs three tests, one each
 835             # that fails, errors and succeeds
 836
 837         def test_sample_method_args(self):
 838             """sample-script.py foo"""
 839             # sample that will run just one test.
 840
 841     def test_construct(self):
 842         test = self.SampleExecTestCase("test_sample_method")
 843         self.assertEqual(test.script,
 844                          subunit.join_dir(__file__, 'sample-script.py'))
 845
 846     def test_args(self):
 847         result = unittest.TestResult()
 848         test = self.SampleExecTestCase("test_sample_method_args")
 849         test.run(result)
 850         self.assertEqual(1, result.testsRun)
 851
 852     def test_run(self):
 853         runner = MockTestProtocolServerClient()
 854         test = self.SampleExecTestCase("test_sample_method")
 855         test.run(runner)
 856         mcdonald = subunit.RemotedTestCase("old mcdonald")
 857         bing = subunit.RemotedTestCase("bing crosby")
 858         an_error = subunit.RemotedTestCase("an error")
 859         self.assertEqual(runner.error_calls,
 860                          [(an_error, subunit.RemoteError())])
 861         self.assertEqual(runner.failure_calls,
 862                          [(bing,
 863                            subunit.RemoteError(
 864                             "foo.c:53:ERROR invalid state\n"))])
 865         self.assertEqual(runner.start_calls, [mcdonald, bing, an_error])
 866         self.assertEqual(runner.end_calls, [mcdonald, bing, an_error])
 867
 868     def test_debug(self):
 869         test = self.SampleExecTestCase("test_sample_method")
 870         test.debug()
 871
 872     def test_count_test_cases(self):
 873         """TODO run the child process and count responses to determine the count."""
 874
 875     def test_join_dir(self):
 876         sibling = subunit.join_dir(__file__, 'foo')
 877         expected = '%s/foo' % (os.path.split(__file__)[0],)
 878         self.assertEqual(sibling, expected)
 879
 880
 881 class DoExecTestCase(subunit.ExecTestCase):
 882
 883     def test_working_script(self):
 884         """sample-two-script.py"""
 885
 886
 887 class TestIsolatedTestCase(unittest.TestCase):
 888
 889     class SampleIsolatedTestCase(subunit.IsolatedTestCase):
 890
 891         SETUP = False
 892         TEARDOWN = False
 893         TEST = False
 894
 895         def setUp(self):
 896             TestIsolatedTestCase.SampleIsolatedTestCase.SETUP = True
 897
 898         def tearDown(self):
 899             TestIsolatedTestCase.SampleIsolatedTestCase.TEARDOWN = True
 900
 901         def test_sets_global_state(self):
 902             TestIsolatedTestCase.SampleIsolatedTestCase.TEST = True
 903
 904
 905     def test_construct(self):
 906         test = self.SampleIsolatedTestCase("test_sets_global_state")
 907
 908     def test_run(self):
 909         result = unittest.TestResult()
 910         test = self.SampleIsolatedTestCase("test_sets_global_state")
 911         test.run(result)
 912         self.assertEqual(result.testsRun, 1)
 913         self.assertEqual(self.SampleIsolatedTestCase.SETUP, False)
 914         self.assertEqual(self.SampleIsolatedTestCase.TEARDOWN, False)
 915         self.assertEqual(self.SampleIsolatedTestCase.TEST, False)
 916
 917     def test_debug(self):
 918         pass
 919         #test = self.SampleExecTestCase("test_sample_method")
 920         #test.debug()
 921
 922
 923 class TestIsolatedTestSuite(unittest.TestCase):
 924
 925     class SampleTestToIsolate(unittest.TestCase):
 926
 927         SETUP = False
 928         TEARDOWN = False
 929         TEST = False
 930
 931         def setUp(self):
 932             TestIsolatedTestSuite.SampleTestToIsolate.SETUP = True
 933
 934         def tearDown(self):
 935             TestIsolatedTestSuite.SampleTestToIsolate.TEARDOWN = True
 936
 937         def test_sets_global_state(self):
 938             TestIsolatedTestSuite.SampleTestToIsolate.TEST = True
 939
 940
 941     def test_construct(self):
 942         suite = subunit.IsolatedTestSuite()
 943
 944     def test_run(self):
 945         result = unittest.TestResult()
 946         suite = subunit.IsolatedTestSuite()
 947         sub_suite = unittest.TestSuite()
 948         sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
 949         sub_suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
 950         suite.addTest(sub_suite)
 951         suite.addTest(self.SampleTestToIsolate("test_sets_global_state"))
 952         suite.run(result)
 953         self.assertEqual(result.testsRun, 3)
 954         self.assertEqual(self.SampleTestToIsolate.SETUP, False)
 955         self.assertEqual(self.SampleTestToIsolate.TEARDOWN, False)
 956         self.assertEqual(self.SampleTestToIsolate.TEST, False)
 957
 958
 959 class TestTestProtocolClient(unittest.TestCase):
 960
 961     def setUp(self):
 962         self.io = StringIO()
 963         self.protocol = subunit.TestProtocolClient(self.io)
 964         self.test = TestTestProtocolClient("test_start_test")
 965
 966     def test_start_test(self):
 967         """Test startTest on a TestProtocolClient."""
 968         self.protocol.startTest(self.test)
 969         self.assertEqual(self.io.getvalue(), "test: %s\n" % self.test.id())
 970
 971     def test_stop_test(self):
 972         # stopTest doesn't output anything.
 973         self.protocol.stopTest(self.test)
 974         self.assertEqual(self.io.getvalue(), "")
 975
 976     def test_add_success(self):
 977         """Test addSuccess on a TestProtocolClient."""
 978         self.protocol.addSuccess(self.test)
 979         self.assertEqual(
 980             self.io.getvalue(), "successful: %s\n" % self.test.id())
 981
 982     def test_add_failure(self):
 983         """Test addFailure on a TestProtocolClient."""
 984         self.protocol.addFailure(
 985             self.test, subunit.RemoteError("boo qux"))
 986         self.assertEqual(
 987             self.io.getvalue(),
 988             'failure: %s [\nRemoteException: boo qux\n]\n' % self.test.id())
 989
 990     def test_add_error(self):
 991         """Test stopTest on a TestProtocolClient."""
 992         self.protocol.addError(
 993             self.test, subunit.RemoteError("phwoar crikey"))
 994         self.assertEqual(
 995             self.io.getvalue(),
 996             'error: %s [\n'
 997             "RemoteException: phwoar crikey\n"
 998             "]\n" % self.test.id())
 999
1000     def test_add_skip(self):
1001         """Test addSkip on a TestProtocolClient."""
1002         self.protocol.addSkip(
1003             self.test, "Has it really?")
1004         self.assertEqual(
1005             self.io.getvalue(),
1006             'skip: %s [\nHas it really?\n]\n' % self.test.id())
1007
1008     def test_time(self):
1009         # Calling time() outputs a time signal immediately.
1010         self.protocol.time(
1011             datetime.datetime(2009,10,11,12,13,14,15, iso8601.Utc()))
1012         self.assertEqual(
1013             "time: 2009-10-11 12:13:14.000015Z\n",
1014             self.io.getvalue())
1015
1016
1017 def test_suite():
1018     loader = subunit.tests.TestUtil.TestLoader()
1019     result = loader.loadTestsFromName(__name__)
1020     return result