3f36499562c8f858317dc666a41de2b8158cb505
[third_party/subunit] / lib / subunit / tests / test_test_protocol.py
1 #
2 #  subunit: extensions to python unittest to get test results from subprocesses.
3 #  Copyright (C) 2005  Robert Collins <robertc@robertcollins.net>
4 #
5 #  This program is free software; you can redistribute it and/or modify
6 #  it under the terms of the GNU General Public License as published by
7 #  the Free Software Foundation; either version 2 of the License, or
8 #  (at your option) any later version.
9 #
10 #  This program is distributed in the hope that it will be useful,
11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 #  GNU General Public License for more details.
14 #
15 #  You should have received a copy of the GNU General Public License
16 #  along with this program; if not, write to the Free Software
17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18 #
19
20 import unittest
21 from StringIO import StringIO
22 import subunit
23 import sys
24
25 try:
26     class MockTestProtocolServerClient(object):
27         """A mock protocol server client to test callbacks."""
28
29         def __init__(self):
30             self.end_calls = []
31             self.error_calls = []
32             self.failure_calls = []
33             self.start_calls = []
34             self.success_calls = []
35             super(MockTestProtocolServerClient, self).__init__()
36
37         def addError(self, test, error):
38             self.error_calls.append((test, error))
39
40         def addFailure(self, test, error):
41             self.failure_calls.append((test, error))
42
43         def addSuccess(self, test):
44             self.success_calls.append(test)
45
46         def endTest(self, test):
47             self.end_calls.append(test)
48             
49         def startTest(self, test):
50             self.start_calls.append(test)
51
52 except AttributeError:
53     MockTestProtocolServer = None
54
55
56 class TestMockTestProtocolServer(unittest.TestCase):
57
58     def test_start_test(self):
59         protocol = MockTestProtocolServerClient()
60         protocol.startTest(subunit.RemotedTestCase("test old mcdonald"))
61         self.assertEqual(protocol.start_calls,
62                          [subunit.RemotedTestCase("test old mcdonald")])
63         self.assertEqual(protocol.end_calls, [])
64         self.assertEqual(protocol.error_calls, [])
65         self.assertEqual(protocol.failure_calls, [])
66         self.assertEqual(protocol.success_calls, [])
67
68     def test_add_error(self):
69         protocol = MockTestProtocolServerClient()
70         protocol.addError(subunit.RemotedTestCase("old mcdonald"), 
71                           subunit.RemoteError("omg it works"))
72         self.assertEqual(protocol.start_calls, [])
73         self.assertEqual(protocol.end_calls, [])
74         self.assertEqual(protocol.error_calls, [(
75                             subunit.RemotedTestCase("old mcdonald"),
76                             subunit.RemoteError("omg it works"))])
77         self.assertEqual(protocol.failure_calls, [])
78         self.assertEqual(protocol.success_calls, [])
79         
80     def test_add_failure(self):
81         protocol = MockTestProtocolServerClient()
82         protocol.addFailure(subunit.RemotedTestCase("old mcdonald"),
83                             subunit.RemoteError("omg it works"))
84         self.assertEqual(protocol.start_calls, [])
85         self.assertEqual(protocol.end_calls, [])
86         self.assertEqual(protocol.error_calls, [])
87         self.assertEqual(protocol.failure_calls, [
88                             (subunit.RemotedTestCase("old mcdonald"),
89                              subunit.RemoteError("omg it works"))])
90         self.assertEqual(protocol.success_calls, [])
91
92     def test_add_success(self):
93         protocol = MockTestProtocolServerClient()
94         protocol.addSuccess(subunit.RemotedTestCase("test old mcdonald"))
95         self.assertEqual(protocol.start_calls, [])
96         self.assertEqual(protocol.end_calls, [])
97         self.assertEqual(protocol.error_calls, [])
98         self.assertEqual(protocol.failure_calls, [])
99         self.assertEqual(protocol.success_calls, 
100                          [subunit.RemotedTestCase("test old mcdonald")])
101         
102     def test_end_test(self):
103         protocol = MockTestProtocolServerClient()
104         protocol.endTest(subunit.RemotedTestCase("test old mcdonald"))
105         self.assertEqual(protocol.end_calls,
106                          [subunit.RemotedTestCase("test old mcdonald")])
107         self.assertEqual(protocol.error_calls, [])
108         self.assertEqual(protocol.failure_calls, [])
109         self.assertEqual(protocol.success_calls, [])
110         self.assertEqual(protocol.start_calls, [])
111
112 class TestTestImports(unittest.TestCase):
113     
114     def test_imports(self):
115         from subunit import TestProtocolServer
116         from subunit import RemotedTestCase
117         from subunit import RemoteError
118
119
120 class TestTestProtocolServerPipe(unittest.TestCase):
121
122     def test_story(self):
123         client = MockTestProtocolServerClient()
124         protocol = subunit.TestProtocolServer(client)
125         pipe = StringIO("test old mcdonald\n"
126                         "success old mcdonald\n"
127                         "test bing crosby\n"
128                         "failure bing crosby [\n"
129                         "foo.c:53:ERROR invalid state\n"
130                         "]\n"
131                         "test an error\n"
132                         "error an error\n")
133         protocol.readFrom(pipe)
134         mcdonald = subunit.RemotedTestCase("old mcdonald")
135         bing = subunit.RemotedTestCase("bing crosby")
136         an_error = subunit.RemotedTestCase("an error")
137         self.assertEqual(client.start_calls, [mcdonald, bing, an_error])
138         self.assertEqual(client.end_calls, [mcdonald, bing, an_error])
139         self.assertEqual(client.error_calls, 
140                          [(an_error, subunit.RemoteError())])
141         self.assertEqual(client.failure_calls, 
142                          [(bing,
143                            subunit.RemoteError("foo.c:53:ERROR "
144                                                "invalid state\n"))])
145         self.assertEqual(client.success_calls, [mcdonald])
146
147
148 class TestTestProtocolServerStartTest(unittest.TestCase):
149     
150     def setUp(self):
151         self.client = MockTestProtocolServerClient()
152         self.protocol = subunit.TestProtocolServer(self.client)
153     
154     def test_start_test(self):
155         self.protocol.lineReceived("test old mcdonald\n")
156         self.assertEqual(self.client.start_calls,
157                          [subunit.RemotedTestCase("old mcdonald")])
158
159     def test_start_testing(self):
160         self.protocol.lineReceived("testing old mcdonald\n")
161         self.assertEqual(self.client.start_calls,
162                          [subunit.RemotedTestCase("old mcdonald")])
163
164     def test_start_test_colon(self):
165         self.protocol.lineReceived("test: old mcdonald\n")
166         self.assertEqual(self.client.start_calls,
167                          [subunit.RemotedTestCase("old mcdonald")])
168
169     def test_start_testing_colon(self):
170         self.protocol.lineReceived("testing: old mcdonald\n")
171         self.assertEqual(self.client.start_calls,
172                          [subunit.RemotedTestCase("old mcdonald")])
173
174
175 class TestTestProtocolServerPassThrough(unittest.TestCase):
176
177     def setUp(self):
178         from StringIO import StringIO
179         self.real_stdout = sys.stdout
180         self.stdout = StringIO()
181         sys.stdout = self.stdout
182         self.test = subunit.RemotedTestCase("old mcdonald")
183         self.client = MockTestProtocolServerClient()
184         self.protocol = subunit.TestProtocolServer(self.client)
185         
186     def tearDown(self):
187         sys.stdout = self.real_stdout
188
189     def keywords_before_test(self):
190         self.protocol.lineReceived("failure a\n")
191         self.protocol.lineReceived("failure: a\n")
192         self.protocol.lineReceived("error a\n")
193         self.protocol.lineReceived("error: a\n")
194         self.protocol.lineReceived("success a\n")
195         self.protocol.lineReceived("success: a\n")
196         self.protocol.lineReceived("successful a\n")
197         self.protocol.lineReceived("successful: a\n")
198         self.protocol.lineReceived("]\n")
199         self.assertEqual(self.stdout.getvalue(), "failure a\n"
200                                                  "failure: a\n"
201                                                  "error a\n"
202                                                  "error: a\n"
203                                                  "success a\n"
204                                                  "success: a\n"
205                                                  "successful a\n"
206                                                  "successful: a\n"
207                                                  "]\n")
208
209     def test_keywords_before_test(self):
210         self.keywords_before_test()
211         self.assertEqual(self.client.start_calls, [])
212         self.assertEqual(self.client.error_calls, [])
213         self.assertEqual(self.client.failure_calls, [])
214         self.assertEqual(self.client.success_calls, [])
215
216     def test_keywords_after_error(self):
217         self.protocol.lineReceived("test old mcdonald\n")
218         self.protocol.lineReceived("error old mcdonald\n")
219         self.keywords_before_test()
220         self.assertEqual(self.client.start_calls, [self.test])
221         self.assertEqual(self.client.end_calls, [self.test])
222         self.assertEqual(self.client.error_calls, 
223                          [(self.test, subunit.RemoteError(""))])
224         self.assertEqual(self.client.failure_calls, [])
225         self.assertEqual(self.client.success_calls, [])
226         
227     def test_keywords_after_failure(self):
228         self.protocol.lineReceived("test old mcdonald\n")
229         self.protocol.lineReceived("failure old mcdonald\n")
230         self.keywords_before_test()
231         self.assertEqual(self.client.start_calls, [self.test])
232         self.assertEqual(self.client.end_calls, [self.test])
233         self.assertEqual(self.client.error_calls, [])
234         self.assertEqual(self.client.failure_calls, 
235                          [(self.test, subunit.RemoteError())])
236         self.assertEqual(self.client.success_calls, [])
237         
238     def test_keywords_after_success(self):
239         self.protocol.lineReceived("test old mcdonald\n")
240         self.protocol.lineReceived("success old mcdonald\n")
241         self.keywords_before_test()
242         self.assertEqual(self.client.start_calls, [self.test])
243         self.assertEqual(self.client.end_calls, [self.test])
244         self.assertEqual(self.client.error_calls, [])
245         self.assertEqual(self.client.failure_calls, [])
246         self.assertEqual(self.client.success_calls, [self.test])
247
248     def test_keywords_after_test(self):
249         self.protocol.lineReceived("test old mcdonald\n")
250         self.protocol.lineReceived("test old mcdonald\n")
251         self.protocol.lineReceived("failure a\n")
252         self.protocol.lineReceived("failure: a\n")
253         self.protocol.lineReceived("error a\n")
254         self.protocol.lineReceived("error: a\n")
255         self.protocol.lineReceived("success a\n")
256         self.protocol.lineReceived("success: a\n")
257         self.protocol.lineReceived("successful a\n")
258         self.protocol.lineReceived("successful: a\n")
259         self.protocol.lineReceived("]\n")
260         self.protocol.lineReceived("failure old mcdonald\n")
261         self.assertEqual(self.stdout.getvalue(), "test old mcdonald\n"
262                                                  "failure a\n"
263                                                  "failure: a\n"
264                                                  "error a\n"
265                                                  "error: a\n"
266                                                  "success a\n"
267                                                  "success: a\n"
268                                                  "successful a\n"
269                                                  "successful: a\n"
270                                                  "]\n")
271         self.assertEqual(self.client.start_calls, [self.test])
272         self.assertEqual(self.client.end_calls, [self.test])
273         self.assertEqual(self.client.failure_calls, 
274                          [(self.test, subunit.RemoteError())])
275         self.assertEqual(self.client.error_calls, [])
276         self.assertEqual(self.client.success_calls, [])
277
278     def test_keywords_during_failure(self):
279         self.protocol.lineReceived("test old mcdonald\n")
280         self.protocol.lineReceived("failure: old mcdonald [\n")
281         self.protocol.lineReceived("test old mcdonald\n")
282         self.protocol.lineReceived("failure a\n")
283         self.protocol.lineReceived("failure: a\n")
284         self.protocol.lineReceived("error a\n")
285         self.protocol.lineReceived("error: a\n")
286         self.protocol.lineReceived("success a\n")
287         self.protocol.lineReceived("success: a\n")
288         self.protocol.lineReceived("successful a\n")
289         self.protocol.lineReceived("successful: a\n")
290         self.protocol.lineReceived(" ]\n")
291         self.protocol.lineReceived("]\n")
292         self.assertEqual(self.stdout.getvalue(), "")
293         self.assertEqual(self.client.start_calls, [self.test])
294         self.assertEqual(self.client.failure_calls, 
295                          [(self.test, subunit.RemoteError("test old mcdonald\n"
296                                                   "failure a\n"
297                                                   "failure: a\n"
298                                                   "error a\n"
299                                                   "error: a\n"
300                                                   "success a\n"
301                                                   "success: a\n"
302                                                   "successful a\n"
303                                                   "successful: a\n"
304                                                   "]\n"))])
305         self.assertEqual(self.client.end_calls, [self.test])
306         self.assertEqual(self.client.error_calls, [])
307         self.assertEqual(self.client.success_calls, [])
308
309
310 class TestTestProtocolServerLostConnection(unittest.TestCase):
311     
312     def setUp(self):
313         self.client = MockTestProtocolServerClient()
314         self.protocol = subunit.TestProtocolServer(self.client)
315         self.test = subunit.RemotedTestCase("old mcdonald")
316
317     def test_lost_connection_no_input(self):
318         self.protocol.lostConnection()
319         self.assertEqual(self.client.start_calls, [])
320         self.assertEqual(self.client.error_calls, [])
321         self.assertEqual(self.client.failure_calls, [])
322         self.assertEqual(self.client.success_calls, [])
323
324     def test_lost_connection_after_start(self):
325         self.protocol.lineReceived("test old mcdonald\n")
326         self.protocol.lostConnection()
327         self.assertEqual(self.client.start_calls, [self.test])
328         self.assertEqual(self.client.end_calls, [self.test])
329         self.assertEqual(self.client.error_calls, [
330             (self.test, subunit.RemoteError("lost connection during "
331                                             "test 'old mcdonald'"))])
332         self.assertEqual(self.client.failure_calls, [])
333         self.assertEqual(self.client.success_calls, [])
334
335     def test_lost_connected_after_error(self):
336         self.protocol.lineReceived("test old mcdonald\n")
337         self.protocol.lineReceived("error old mcdonald\n")
338         self.protocol.lostConnection()
339         self.assertEqual(self.client.start_calls, [self.test])
340         self.assertEqual(self.client.failure_calls, [])
341         self.assertEqual(self.client.end_calls, [self.test])
342         self.assertEqual(self.client.error_calls, [
343             (self.test, subunit.RemoteError(""))])
344         self.assertEqual(self.client.success_calls, [])
345         
346     def test_lost_connection_during_error(self):
347         self.protocol.lineReceived("test old mcdonald\n")
348         self.protocol.lineReceived("error old mcdonald [\n")
349         self.protocol.lostConnection()
350         self.assertEqual(self.client.start_calls, [self.test])
351         self.assertEqual(self.client.end_calls, [self.test])
352         self.assertEqual(self.client.error_calls, [
353             (self.test, subunit.RemoteError("lost connection during error "
354                                             "report of test 'old mcdonald'"))])
355         self.assertEqual(self.client.failure_calls, [])
356         self.assertEqual(self.client.success_calls, [])
357
358     def test_lost_connected_after_failure(self):
359         self.protocol.lineReceived("test old mcdonald\n")
360         self.protocol.lineReceived("failure old mcdonald\n")
361         self.protocol.lostConnection()
362         test = subunit.RemotedTestCase("old mcdonald")
363         self.assertEqual(self.client.start_calls, [self.test])
364         self.assertEqual(self.client.end_calls, [self.test])
365         self.assertEqual(self.client.error_calls, [])
366         self.assertEqual(self.client.failure_calls, 
367                          [(self.test, subunit.RemoteError())])
368         self.assertEqual(self.client.success_calls, [])
369         
370     def test_lost_connection_during_failure(self):
371         self.protocol.lineReceived("test old mcdonald\n")
372         self.protocol.lineReceived("failure old mcdonald [\n")
373         self.protocol.lostConnection()
374         self.assertEqual(self.client.start_calls, [self.test])
375         self.assertEqual(self.client.end_calls, [self.test])
376         self.assertEqual(self.client.error_calls, 
377                          [(self.test,
378                            subunit.RemoteError("lost connection during "
379                                                "failure report"
380                                                " of test 'old mcdonald'"))])
381         self.assertEqual(self.client.failure_calls, [])
382         self.assertEqual(self.client.success_calls, [])
383
384     def test_lost_connection_after_success(self):
385         self.protocol.lineReceived("test old mcdonald\n")
386         self.protocol.lineReceived("success old mcdonald\n")
387         self.protocol.lostConnection()
388         self.assertEqual(self.client.start_calls, [self.test])
389         self.assertEqual(self.client.end_calls, [self.test])
390         self.assertEqual(self.client.error_calls, [])
391         self.assertEqual(self.client.failure_calls, [])
392         self.assertEqual(self.client.success_calls, [self.test])
393
394
395 class TestTestProtocolServerAddError(unittest.TestCase):
396     
397     def setUp(self):
398         self.client = MockTestProtocolServerClient()
399         self.protocol = subunit.TestProtocolServer(self.client)
400         self.protocol.lineReceived("test mcdonalds farm\n")
401         self.test = subunit.RemotedTestCase("mcdonalds farm")
402
403     def simple_error_keyword(self, keyword):
404         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
405         self.assertEqual(self.client.start_calls, [self.test])
406         self.assertEqual(self.client.end_calls, [self.test])
407         self.assertEqual(self.client.error_calls, [
408             (self.test, subunit.RemoteError(""))])
409         self.assertEqual(self.client.failure_calls, [])
410
411     def test_simple_error(self):
412         self.simple_error_keyword("error")
413
414     def test_simple_error_colon(self):
415         self.simple_error_keyword("error:")
416
417     def test_error_empty_message(self):
418         self.protocol.lineReceived("error mcdonalds farm [\n")
419         self.protocol.lineReceived("]\n")
420         self.assertEqual(self.client.start_calls, [self.test])
421         self.assertEqual(self.client.end_calls, [self.test])
422         self.assertEqual(self.client.error_calls, [
423             (self.test, subunit.RemoteError(""))])
424         self.assertEqual(self.client.failure_calls, [])
425
426     def error_quoted_bracket(self, keyword):
427         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
428         self.protocol.lineReceived(" ]\n")
429         self.protocol.lineReceived("]\n")
430         self.assertEqual(self.client.start_calls, [self.test])
431         self.assertEqual(self.client.end_calls, [self.test])
432         self.assertEqual(self.client.error_calls, [
433             (self.test, subunit.RemoteError("]\n"))])
434         self.assertEqual(self.client.failure_calls, [])
435
436     def test_error_quoted_bracket(self):
437         self.error_quoted_bracket("error")
438
439     def test_error_colon_quoted_bracket(self):
440         self.error_quoted_bracket("error:")
441
442
443 class TestTestProtocolServerAddFailure(unittest.TestCase):
444     
445     def setUp(self):
446         self.client = MockTestProtocolServerClient()
447         self.protocol = subunit.TestProtocolServer(self.client)
448         self.protocol.lineReceived("test mcdonalds farm\n")
449         self.test = subunit.RemotedTestCase("mcdonalds farm")
450
451     def simple_failure_keyword(self, keyword):
452         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
453         self.assertEqual(self.client.start_calls, [self.test])
454         self.assertEqual(self.client.end_calls, [self.test])
455         self.assertEqual(self.client.error_calls, [])
456         self.assertEqual(self.client.failure_calls, 
457                          [(self.test, subunit.RemoteError())])
458
459     def test_simple_failure(self):
460         self.simple_failure_keyword("failure")
461
462     def test_simple_failure_colon(self):
463         self.simple_failure_keyword("failure:")
464
465     def test_failure_empty_message(self):
466         self.protocol.lineReceived("failure mcdonalds farm [\n")
467         self.protocol.lineReceived("]\n")
468         self.assertEqual(self.client.start_calls, [self.test])
469         self.assertEqual(self.client.end_calls, [self.test])
470         self.assertEqual(self.client.error_calls, [])
471         self.assertEqual(self.client.failure_calls, 
472                          [(self.test, subunit.RemoteError())])
473
474     def failure_quoted_bracket(self, keyword):
475         self.protocol.lineReceived("%s mcdonalds farm [\n" % keyword)
476         self.protocol.lineReceived(" ]\n")
477         self.protocol.lineReceived("]\n")
478         self.assertEqual(self.client.start_calls, [self.test])
479         self.assertEqual(self.client.end_calls, [self.test])
480         self.assertEqual(self.client.error_calls, [])
481         self.assertEqual(self.client.failure_calls, 
482                          [(self.test, subunit.RemoteError("]\n"))])
483
484     def test_failure_quoted_bracket(self):
485         self.failure_quoted_bracket("failure")
486
487     def test_failure_colon_quoted_bracket(self):
488         self.failure_quoted_bracket("failure:")
489
490
491 class TestTestProtocolServerAddSuccess(unittest.TestCase):
492     
493     def setUp(self):
494         self.client = MockTestProtocolServerClient()
495         self.protocol = subunit.TestProtocolServer(self.client)
496         self.protocol.lineReceived("test mcdonalds farm\n")
497         self.test = subunit.RemotedTestCase("mcdonalds farm")
498
499     def simple_success_keyword(self, keyword):
500         self.protocol.lineReceived("%s mcdonalds farm\n" % keyword)
501         self.assertEqual(self.client.start_calls, [self.test])
502         self.assertEqual(self.client.end_calls, [self.test])
503         self.assertEqual(self.client.error_calls, [])
504         self.assertEqual(self.client.success_calls, [self.test])
505
506     def test_simple_success(self):
507         self.simple_success_keyword("failure")
508
509     def test_simple_success_colon(self):
510         self.simple_success_keyword("failure:")
511
512     def test_simple_success(self):
513         self.simple_success_keyword("successful")
514
515     def test_simple_success_colon(self):
516         self.simple_success_keyword("successful:")
517
518
519 class TestRemotedTestCase(unittest.TestCase):
520
521     def test_simple(self):
522         test = subunit.RemotedTestCase("A test description")
523         self.assertRaises(NotImplementedError, test.setUp)
524         self.assertRaises(NotImplementedError, test.tearDown)
525         self.assertEqual("A test description",
526                          test.shortDescription())
527         self.assertEqual("subunit.RemotedTestCase.A test description",
528                          test.id())
529         self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
530         self.assertEqual("<subunit.RemotedTestCase description="
531                          "'A test description'>", "%r" % test)
532         result = unittest.TestResult()
533         test.run(result)
534         self.assertEqual([(test, "RemoteError:\n"
535                                  "Cannot run RemotedTestCases.\n\n")],
536                          result.errors)
537         self.assertEqual(1, result.testsRun)
538         another_test = subunit.RemotedTestCase("A test description")
539         self.assertEqual(test, another_test)
540         different_test = subunit.RemotedTestCase("ofo")
541         self.assertNotEqual(test, different_test)
542         self.assertNotEqual(another_test, different_test)
543
544
545 class TestRemoteError(unittest.TestCase):
546
547     def test_eq(self):
548         error = subunit.RemoteError("Something went wrong")
549         another_error = subunit.RemoteError("Something went wrong")
550         different_error = subunit.RemoteError("boo!")
551         self.assertEqual(error, another_error)
552         self.assertNotEqual(error, different_error)
553         self.assertNotEqual(different_error, another_error)
554
555     def test_empty_constructor(self):
556         self.assertEqual(subunit.RemoteError(), subunit.RemoteError(""))
557
558 def test_suite():
559     loader = subunit.tests.TestUtil.TestLoader()
560     result = loader.loadTestsFromName(__name__)
561     return result