2 # subunit: extensions to python unittest to get test results from subprocesses.
3 # Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 from StringIO import StringIO
26 class MockTestProtocolServerClient(object):
27 """A mock protocol server client to test callbacks."""
32 self.failure_calls = []
34 self.success_calls = []
35 super(MockTestProtocolServerClient, self).__init__()
37 def addError(self, test, error):
38 self.error_calls.append((test, error))
40 def addFailure(self, test, error):
41 self.failure_calls.append((test, error))
43 def addSuccess(self, test):
44 self.success_calls.append(test)
46 def endTest(self, test):
47 self.end_calls.append(test)
49 def startTest(self, test):
50 self.start_calls.append(test)
52 except AttributeError:
53 MockTestProtocolServer = None
56 class TestMockTestProtocolServer(unittest.TestCase):
58 def test_start_test(self):
59 protocol = MockTestProtocolServerClient()
60 protocol.startTest(subunit.RemotedTestCase("test old mcdonald"))
61 self.assertEqual(protocol.start_calls,
62 [subunit.RemotedTestCase("test old mcdonald")])
63 self.assertEqual(protocol.end_calls, [])
64 self.assertEqual(protocol.error_calls, [])
65 self.assertEqual(protocol.failure_calls, [])
66 self.assertEqual(protocol.success_calls, [])
68 def test_add_error(self):
69 protocol = MockTestProtocolServerClient()
70 protocol.addError(subunit.RemotedTestCase("old mcdonald"),
71 subunit.RemoteError("omg it works"))
72 self.assertEqual(protocol.start_calls, [])
73 self.assertEqual(protocol.end_calls, [])
74 self.assertEqual(protocol.error_calls, [(
75 subunit.RemotedTestCase("old mcdonald"),
76 subunit.RemoteError("omg it works"))])
77 self.assertEqual(protocol.failure_calls, [])
78 self.assertEqual(protocol.success_calls, [])
80 def test_add_failure(self):
81 protocol = MockTestProtocolServerClient()
82 protocol.addFailure(subunit.RemotedTestCase("old mcdonald"),
83 subunit.RemoteError("omg it works"))
84 self.assertEqual(protocol.start_calls, [])
85 self.assertEqual(protocol.end_calls, [])
86 self.assertEqual(protocol.error_calls, [])
87 self.assertEqual(protocol.failure_calls, [
88 (subunit.RemotedTestCase("old mcdonald"),
89 subunit.RemoteError("omg it works"))])
90 self.assertEqual(protocol.success_calls, [])
92 def test_add_success(self):
93 protocol = MockTestProtocolServerClient()
94 protocol.addSuccess(subunit.RemotedTestCase("test old mcdonald"))
95 self.assertEqual(protocol.start_calls, [])
96 self.assertEqual(protocol.end_calls, [])
97 self.assertEqual(protocol.error_calls, [])
98 self.assertEqual(protocol.failure_calls, [])
99 self.assertEqual(protocol.success_calls,
100 [subunit.RemotedTestCase("test old mcdonald")])
102 def test_end_test(self):
103 protocol = MockTestProtocolServerClient()
104 protocol.endTest(subunit.RemotedTestCase("test old mcdonald"))
105 self.assertEqual(protocol.end_calls,
106 [subunit.RemotedTestCase("test old mcdonald")])
107 self.assertEqual(protocol.error_calls, [])
108 self.assertEqual(protocol.failure_calls, [])
109 self.assertEqual(protocol.success_calls, [])
110 self.assertEqual(protocol.start_calls, [])
112 class TestTestImports(unittest.TestCase):
114 def test_imports(self):
115 from subunit import TestProtocolServer
116 from subunit import RemotedTestCase
117 from subunit import RemoteError
120 class TestTestProtocolServerPipe(unittest.TestCase):
122 def test_story(self):
123 client = MockTestProtocolServerClient()
124 protocol = subunit.TestProtocolServer(client)
125 pipe = StringIO("test old mcdonald\n"
126 "success old mcdonald\n"
128 "failure bing crosby [\n"
129 "foo.c:53:ERROR invalid state\n"
133 protocol.readFrom(pipe)
134 mcdonald = subunit.RemotedTestCase("old mcdonald")
135 bing = subunit.RemotedTestCase("bing crosby")
136 an_error = subunit.RemotedTestCase("an error")
137 self.assertEqual(client.start_calls, [mcdonald, bing, an_error])
138 self.assertEqual(client.end_calls, [mcdonald, bing, an_error])
139 self.assertEqual(client.error_calls,
140 [(an_error, subunit.RemoteError())])
141 self.assertEqual(client.failure_calls,
143 subunit.RemoteError("foo.c:53:ERROR "
144 "invalid state\n"))])
145 self.assertEqual(client.success_calls, [mcdonald])
148 class TestTestProtocolServerStartTest(unittest.TestCase):
151 self.client = MockTestProtocolServerClient()
152 self.protocol = subunit.TestProtocolServer(self.client)
154 def test_start_test(self):
155 self.protocol.lineReceived("test old mcdonald\n")
156 self.assertEqual(self.client.start_calls,
157 [subunit.RemotedTestCase("old mcdonald")])
159 def test_start_testing(self):
160 self.protocol.lineReceived("testing old mcdonald\n")
161 self.assertEqual(self.client.start_calls,
162 [subunit.RemotedTestCase("old mcdonald")])
164 def test_start_test_colon(self):
165 self.protocol.lineReceived("test: old mcdonald\n")
166 self.assertEqual(self.client.start_calls,
167 [subunit.RemotedTestCase("old mcdonald")])
169 def test_start_testing_colon(self):
170 self.protocol.lineReceived("testing: old mcdonald\n")
171 self.assertEqual(self.client.start_calls,
172 [subunit.RemotedTestCase("old mcdonald")])
175 class TestTestProtocolServerPassThrough(unittest.TestCase):
178 from StringIO import StringIO
179 self.real_stdout = sys.stdout
180 self.stdout = StringIO()
181 sys.stdout = self.stdout
182 self.test = subunit.RemotedTestCase("old mcdonald")
183 self.client = MockTestProtocolServerClient()
184 self.protocol = subunit.TestProtocolServer(self.client)
187 sys.stdout = self.real_stdout
189 def keywords_before_test(self):
190 self.protocol.lineReceived("failure a\n")
191 self.protocol.lineReceived("failure: a\n")
192 self.protocol.lineReceived("error a\n")
193 self.protocol.lineReceived("error: a\n")
194 self.protocol.lineReceived("success a\n")
195 self.protocol.lineReceived("success: a\n")
196 self.protocol.lineReceived("successful a\n")
197 self.protocol.lineReceived("successful: a\n")
198 self.protocol.lineReceived("]\n")
199 self.assertEqual(self.stdout.getvalue(), "failure a\n"
209 def test_keywords_before_test(self):
210 self.keywords_before_test()
211 self.assertEqual(self.client.start_calls, [])
212 self.assertEqual(self.client.error_calls, [])
213 self.assertEqual(self.client.failure_calls, [])
214 self.assertEqual(self.client.success_calls, [])
216 def test_keywords_after_error(self):
217 self.protocol.lineReceived("test old mcdonald\n")
218 self.protocol.lineReceived("error old mcdonald\n")
219 self.keywords_before_test()
220 self.assertEqual(self.client.start_calls, [self.test])
221 self.assertEqual(self.client.end_calls, [self.test])
222 self.assertEqual(self.client.error_calls,
223 [(self.test, subunit.RemoteError(""))])
224 self.assertEqual(self.client.failure_calls, [])
225 self.assertEqual(self.client.success_calls, [])
227 def test_keywords_after_failure(self):
228 self.protocol.lineReceived("test old mcdonald\n")
229 self.protocol.lineReceived("failure old mcdonald\n")
230 self.keywords_before_test()
231 self.assertEqual(self.client.start_calls, [self.test])
232 self.assertEqual(self.client.end_calls, [self.test])
233 self.assertEqual(self.client.error_calls, [])
234 self.assertEqual(self.client.failure_calls,
235 [(self.test, subunit.RemoteError())])
236 self.assertEqual(self.client.success_calls, [])
238 def test_keywords_after_success(self):
239 self.protocol.lineReceived("test old mcdonald\n")
240 self.protocol.lineReceived("success old mcdonald\n")
241 self.keywords_before_test()
242 self.assertEqual(self.client.start_calls, [self.test])
243 self.assertEqual(self.client.end_calls, [self.test])
244 self.assertEqual(self.client.error_calls, [])
245 self.assertEqual(self.client.failure_calls, [])
246 self.assertEqual(self.client.success_calls, [self.test])
248 def test_keywords_after_test(self):
249 self.protocol.lineReceived("test old mcdonald\n")
250 self.protocol.lineReceived("test old mcdonald\n")
251 self.protocol.lineReceived("failure a\n")
252 self.protocol.lineReceived("failure: a\n")
253 self.protocol.lineReceived("error a\n")
254 self.protocol.lineReceived("error: a\n")
255 self.protocol.lineReceived("success a\n")
256 self.protocol.lineReceived("success: a\n")
257 self.protocol.lineReceived("successful a\n")
258 self.protocol.lineReceived("successful: a\n")
259 self.protocol.lineReceived("]\n")
260 self.protocol.lineReceived("failure old mcdonald\n")
261 self.assertEqual(self.stdout.getvalue(), "test old mcdonald\n"
271 self.assertEqual(self.client.start_calls, [self.test])
272 self.assertEqual(self.client.end_calls, [self.test])
273 self.assertEqual(self.client.failure_calls,
274 [(self.test, subunit.RemoteError())])
275 self.assertEqual(self.client.error_calls, [])
276 self.assertEqual(self.client.success_calls, [])
278 def test_keywords_during_failure(self):
279 self.protocol.lineReceived("test old mcdonald\n")
280 self.protocol.lineReceived("failure: old mcdonald [\n")
281 self.protocol.lineReceived("test old mcdonald\n")
282 self.protocol.lineReceived("failure a\n")
283 self.protocol.lineReceived("failure: a\n")
284 self.protocol.lineReceived("error a\n")
285 self.protocol.lineReceived("error: a\n")
286 self.protocol.lineReceived("success a\n")
287 self.protocol.lineReceived("success: a\n")
288 self.protocol.lineReceived("successful a\n")
289 self.protocol.lineReceived("successful: a\n")
290 self.protocol.lineReceived(" ]\n")
291 self.protocol.lineReceived("]\n")
292 self.assertEqual(self.stdout.getvalue(), "")
293 self.assertEqual(self.client.start_calls, [self.test])
294 self.assertEqual(self.client.failure_calls,
295 [(self.test, subunit.RemoteError("test old mcdonald\n"
305 self.assertEqual(self.client.end_calls, [self.test])
306 self.assertEqual(self.client.error_calls, [])
307 self.assertEqual(self.client.success_calls, [])
310 class TestTestProtocolServerLostConnection(unittest.TestCase):
313 self.client = MockTestProtocolServerClient()
314 self.protocol = subunit.TestProtocolServer(self.client)
315 self.test = subunit.RemotedTestCase("old mcdonald")
317 def test_lost_connection_no_input(self):
318 self.protocol.lostConnection()
319 self.assertEqual(self.client.start_calls, [])
320 self.assertEqual(self.client.error_calls, [])
321 self.assertEqual(self.client.failure_calls, [])
322 self.assertEqual(self.client.success_calls, [])
324 def test_lost_connection_after_start(self):
325 self.protocol.lineReceived("test old mcdonald\n")
326 self.protocol.lostConnection()
327 self.assertEqual(self.client.start_calls, [self.test])
328 self.assertEqual(self.client.end_calls, [self.test])
329 self.assertEqual(self.client.error_calls, [
330 (self.test, subunit.RemoteError("lost connection during "
331 "test 'old mcdonald'"))])
332 self.assertEqual(self.client.failure_calls, [])
333 self.assertEqual(self.client.success_calls, [])
335 def test_lost_connected_after_error(self):
336 self.protocol.lineReceived("test old mcdonald\n")
337 self.protocol.lineReceived("error old mcdonald\n")
338 self.protocol.lostConnection()
339 self.assertEqual(self.client.start_calls, [self.test])
340 self.assertEqual(self.client.failure_calls, [])
341 self.assertEqual(self.client.end_calls, [self.test])
342 self.assertEqual(self.client.error_calls, [
343 (self.test, subunit.RemoteError(""))])
344 self.assertEqual(self.client.success_calls, [])
346 def test_lost_connection_during_error(self):
347 self.protocol.lineReceived("test old mcdonald\n")
348 self.protocol.lineReceived("error old mcdonald [\n")
349 self.protocol.lostConnection()
350 self.assertEqual(self.client.start_calls, [self.test])
351 self.assertEqual(self.client.end_calls, [self.test])
352 self.assertEqual(self.client.error_calls, [
353 (self.test, subunit.RemoteError("lost connection during error "
354 "report of test 'old mcdonald'"))])
355 self.assertEqual(self.client.failure_calls, [])
356 self.assertEqual(self.client.success_calls, [])
358 def test_lost_connected_after_failure(self):
359 self.protocol.lineReceived("test old mcdonald\n")
360 self.protocol.lineReceived("failure old mcdonald\n")
361 self.protocol.lostConnection()
362 test = subunit.RemotedTestCase("old mcdonald")
363 self.assertEqual(self.client.start_calls, [self.test])
364 self.assertEqual(self.client.end_calls, [self.test])
365 self.assertEqual(self.client.error_calls, [])
366 self.assertEqual(self.client.failure_calls,
367 [(self.test, subunit.RemoteError())])
368 self.assertEqual(self.client.success_calls, [])
370 def test_lost_connection_during_failure(self):
371 self.protocol.lineReceived("test old mcdonald\n")
372 self.protocol.lineReceived("failure old mcdonald [\n")
373 self.protocol.lostConnection()
374 self.assertEqual(self.client.start_calls, [self.test])
375 self.assertEqual(self.client.end_calls, [self.test])
376 self.assertEqual(self.client.error_calls,
378 subunit.RemoteError("lost connection during "
380 " of test 'old mcdonald'"))])
381 self.assertEqual(self.client.failure_calls, [])
382 self.assertEqual(self.client.success_calls, [])
384 def test_lost_connection_after_success(self):
385 self.protocol.lineReceived("test old mcdonald\n")
386 self.protocol.lineReceived("success old mcdonald\n")
387 self.protocol.lostConnection()
388 self.assertEqual(self.client.start_calls, [self.test])
389 self.assertEqual(self.client.end_calls, [self.test])
390 self.assertEqual(self.client.error_calls, [])
391 self.assertEqual(self.client.failure_calls, [])
392 self.assertEqual(self.client.success_calls, [self.test])
395 class TestTestProtocolServerAddError(unittest.TestCase):
398 self.client = MockTestProtocolServerClient()
399 self.protocol = subunit.TestProtocolServer(self.client)
400 self.protocol.lineReceived("test mcdonalds farm\n")
401 self.test = subunit.RemotedTestCase("mcdonalds farm")
403 def simple_error_keyword(self, keyword):
404 self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
405 self.assertEqual(self.client.start_calls, [self.test])
406 self.assertEqual(self.client.end_calls, [self.test])
407 self.assertEqual(self.client.error_calls, [
408 (self.test, subunit.RemoteError(""))])
409 self.assertEqual(self.client.failure_calls, [])
411 def test_simple_error(self):
412 self.simple_error_keyword("error")
414 def test_simple_error_colon(self):
415 self.simple_error_keyword("error:")
417 def test_error_empty_message(self):
418 self.protocol.lineReceived("error mcdonalds farm [\n")
419 self.protocol.lineReceived("]\n")
420 self.assertEqual(self.client.start_calls, [self.test])
421 self.assertEqual(self.client.end_calls, [self.test])
422 self.assertEqual(self.client.error_calls, [
423 (self.test, subunit.RemoteError(""))])
424 self.assertEqual(self.client.failure_calls, [])
426 def error_quoted_bracket(self, keyword):
427 self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
428 self.protocol.lineReceived(" ]\n")
429 self.protocol.lineReceived("]\n")
430 self.assertEqual(self.client.start_calls, [self.test])
431 self.assertEqual(self.client.end_calls, [self.test])
432 self.assertEqual(self.client.error_calls, [
433 (self.test, subunit.RemoteError("]\n"))])
434 self.assertEqual(self.client.failure_calls, [])
436 def test_error_quoted_bracket(self):
437 self.error_quoted_bracket("error")
439 def test_error_colon_quoted_bracket(self):
440 self.error_quoted_bracket("error:")
443 class TestTestProtocolServerAddFailure(unittest.TestCase):
446 self.client = MockTestProtocolServerClient()
447 self.protocol = subunit.TestProtocolServer(self.client)
448 self.protocol.lineReceived("test mcdonalds farm\n")
449 self.test = subunit.RemotedTestCase("mcdonalds farm")
451 def simple_failure_keyword(self, keyword):
452 self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
453 self.assertEqual(self.client.start_calls, [self.test])
454 self.assertEqual(self.client.end_calls, [self.test])
455 self.assertEqual(self.client.error_calls, [])
456 self.assertEqual(self.client.failure_calls,
457 [(self.test, subunit.RemoteError())])
459 def test_simple_failure(self):
460 self.simple_failure_keyword("failure")
462 def test_simple_failure_colon(self):
463 self.simple_failure_keyword("failure:")
465 def test_failure_empty_message(self):
466 self.protocol.lineReceived("failure mcdonalds farm [\n")
467 self.protocol.lineReceived("]\n")
468 self.assertEqual(self.client.start_calls, [self.test])
469 self.assertEqual(self.client.end_calls, [self.test])
470 self.assertEqual(self.client.error_calls, [])
471 self.assertEqual(self.client.failure_calls,
472 [(self.test, subunit.RemoteError())])
474 def failure_quoted_bracket(self, keyword):
475 self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
476 self.protocol.lineReceived(" ]\n")
477 self.protocol.lineReceived("]\n")
478 self.assertEqual(self.client.start_calls, [self.test])
479 self.assertEqual(self.client.end_calls, [self.test])
480 self.assertEqual(self.client.error_calls, [])
481 self.assertEqual(self.client.failure_calls,
482 [(self.test, subunit.RemoteError("]\n"))])
484 def test_failure_quoted_bracket(self):
485 self.failure_quoted_bracket("failure")
487 def test_failure_colon_quoted_bracket(self):
488 self.failure_quoted_bracket("failure:")
491 class TestTestProtocolServerAddSuccess(unittest.TestCase):
494 self.client = MockTestProtocolServerClient()
495 self.protocol = subunit.TestProtocolServer(self.client)
496 self.protocol.lineReceived("test mcdonalds farm\n")
497 self.test = subunit.RemotedTestCase("mcdonalds farm")
499 def simple_success_keyword(self, keyword):
500 self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
501 self.assertEqual(self.client.start_calls, [self.test])
502 self.assertEqual(self.client.end_calls, [self.test])
503 self.assertEqual(self.client.error_calls, [])
504 self.assertEqual(self.client.success_calls, [self.test])
506 def test_simple_success(self):
507 self.simple_success_keyword("failure")
509 def test_simple_success_colon(self):
510 self.simple_success_keyword("failure:")
512 def test_simple_success(self):
513 self.simple_success_keyword("successful")
515 def test_simple_success_colon(self):
516 self.simple_success_keyword("successful:")
519 class TestRemotedTestCase(unittest.TestCase):
521 def test_simple(self):
522 test = subunit.RemotedTestCase("A test description")
523 self.assertRaises(NotImplementedError, test.setUp)
524 self.assertRaises(NotImplementedError, test.tearDown)
525 self.assertEqual("A test description",
526 test.shortDescription())
527 self.assertEqual("subunit.RemotedTestCase.A test description",
529 self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
530 self.assertEqual("<subunit.RemotedTestCase description="
531 "'A test description'>", "%r" % test)
532 result = unittest.TestResult()
534 self.assertEqual([(test, "RemoteError:\n"
535 "Cannot run RemotedTestCases.\n\n")],
537 self.assertEqual(1, result.testsRun)
538 another_test = subunit.RemotedTestCase("A test description")
539 self.assertEqual(test, another_test)
540 different_test = subunit.RemotedTestCase("ofo")
541 self.assertNotEqual(test, different_test)
542 self.assertNotEqual(another_test, different_test)
545 class TestRemoteError(unittest.TestCase):
548 error = subunit.RemoteError("Something went wrong")
549 another_error = subunit.RemoteError("Something went wrong")
550 different_error = subunit.RemoteError("boo!")
551 self.assertEqual(error, another_error)
552 self.assertNotEqual(error, different_error)
553 self.assertNotEqual(different_error, another_error)
555 def test_empty_constructor(self):
556 self.assertEqual(subunit.RemoteError(), subunit.RemoteError(""))
559 loader = subunit.tests.TestUtil.TestLoader()
560 result = loader.loadTestsFromName(__name__)