2 # subunit: extensions to python unittest to get test results from subprocesses.
3 # Copyright (C) 2005 Robert Collins <robertc@robertcollins.net>
5 # Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
6 # license at the users choice. A copy of both licenses are available in the
7 # project source as Apache-2.0 and BSD. You may not use this file except in
8 # compliance with one of these two licences.
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # license you chose for the specific language governing permissions and
14 # limitations under that license.
17 """Tests for TAP2SubUnit."""
21 from testtools.compat import StringIO
26 class TestTAP2SubUnit(unittest.TestCase):
27 """Tests for TAP2SubUnit.
29 These tests test TAP string data in, and subunit string data out.
30 This is ok because the subunit protocol is intended to be stable,
31 but it might be easier/pithier to write tests against TAP string in,
32 parsed subunit objects out (by hooking the subunit stream to a subunit
38 self.subunit = StringIO()
40 def test_skip_entire_file(self):
42 # 1..- # Skipped: comment
43 # results in a single skipped test.
44 self.tap.write("1..0 # Skipped: entire file skipped\n")
46 result = subunit.TAP2SubUnit(self.tap, self.subunit)
47 self.assertEqual(0, result)
51 "Skipped: entire file skipped",
54 self.subunit.getvalue().splitlines())
56 def test_ok_test_pass(self):
59 # results in a passed test with name 'test 1' (a synthetic name as tap
60 # does not require named fixtures - it is the first test in the tap
62 self.tap.write("ok\n")
64 result = subunit.TAP2SubUnit(self.tap, self.subunit)
65 self.assertEqual(0, result)
70 self.subunit.getvalue().splitlines())
72 def test_ok_test_number_pass(self):
75 # results in a passed test with name 'test 1'
76 self.tap.write("ok 1\n")
78 result = subunit.TAP2SubUnit(self.tap, self.subunit)
79 self.assertEqual(0, result)
84 self.subunit.getvalue().splitlines())
86 def test_ok_test_number_description_pass(self):
88 # ok 1 - There is a description
89 # results in a passed test with name 'test 1 - There is a description'
90 self.tap.write("ok 1 - There is a description\n")
92 result = subunit.TAP2SubUnit(self.tap, self.subunit)
93 self.assertEqual(0, result)
95 "test test 1 - There is a description",
96 "success test 1 - There is a description",
98 self.subunit.getvalue().splitlines())
100 def test_ok_test_description_pass(self):
102 # ok There is a description
103 # results in a passed test with name 'test 1 There is a description'
104 self.tap.write("ok There is a description\n")
106 result = subunit.TAP2SubUnit(self.tap, self.subunit)
107 self.assertEqual(0, result)
109 "test test 1 There is a description",
110 "success test 1 There is a description",
112 self.subunit.getvalue().splitlines())
114 def test_ok_SKIP_skip(self):
117 # results in a skkip test with name 'test 1'
118 self.tap.write("ok # SKIP\n")
120 result = subunit.TAP2SubUnit(self.tap, self.subunit)
121 self.assertEqual(0, result)
126 self.subunit.getvalue().splitlines())
128 def test_ok_skip_number_comment_lowercase(self):
129 self.tap.write("ok 1 # skip no samba environment available, skipping compilation\n")
131 result = subunit.TAP2SubUnit(self.tap, self.subunit)
132 self.assertEqual(0, result)
136 "no samba environment available, skipping compilation",
139 self.subunit.getvalue().splitlines())
141 def test_ok_number_description_SKIP_skip_comment(self):
143 # ok 1 foo # SKIP Not done yet
144 # results in a skip test with name 'test 1 foo' and a log of
146 self.tap.write("ok 1 foo # SKIP Not done yet\n")
148 result = subunit.TAP2SubUnit(self.tap, self.subunit)
149 self.assertEqual(0, result)
156 self.subunit.getvalue().splitlines())
158 def test_ok_SKIP_skip_comment(self):
160 # ok # SKIP Not done yet
161 # results in a skip test with name 'test 1' and a log of Not done yet
162 self.tap.write("ok # SKIP Not done yet\n")
164 result = subunit.TAP2SubUnit(self.tap, self.subunit)
165 self.assertEqual(0, result)
172 self.subunit.getvalue().splitlines())
174 def test_ok_TODO_xfail(self):
177 # results in a xfail test with name 'test 1'
178 self.tap.write("ok # TODO\n")
180 result = subunit.TAP2SubUnit(self.tap, self.subunit)
181 self.assertEqual(0, result)
186 self.subunit.getvalue().splitlines())
188 def test_ok_TODO_xfail_comment(self):
190 # ok # TODO Not done yet
191 # results in a xfail test with name 'test 1' and a log of Not done yet
192 self.tap.write("ok # TODO Not done yet\n")
194 result = subunit.TAP2SubUnit(self.tap, self.subunit)
195 self.assertEqual(0, result)
202 self.subunit.getvalue().splitlines())
204 def test_bail_out_errors(self):
205 # A file with line in it
207 # is treated as an error
208 self.tap.write("ok 1 foo\n")
209 self.tap.write("Bail out! Lifejacket engaged\n")
211 result = subunit.TAP2SubUnit(self.tap, self.subunit)
212 self.assertEqual(0, result)
215 "success test 1 foo",
216 "test Bail out! Lifejacket engaged",
217 "error Bail out! Lifejacket engaged",
219 self.subunit.getvalue().splitlines())
221 def test_missing_test_at_end_with_plan_adds_error(self):
226 # results in three tests, with the third being created
227 self.tap.write('1..3\n')
228 self.tap.write('ok first test\n')
229 self.tap.write('not ok second test\n')
231 result = subunit.TAP2SubUnit(self.tap, self.subunit)
232 self.assertEqual(0, result)
234 'test test 1 first test',
235 'success test 1 first test',
236 'test test 2 second test',
237 'failure test 2 second test',
240 'test missing from TAP output',
243 self.subunit.getvalue().splitlines())
245 def test_missing_test_with_plan_adds_error(self):
249 # not ok 3 third test
250 # results in three tests, with the second being created
251 self.tap.write('1..3\n')
252 self.tap.write('ok first test\n')
253 self.tap.write('not ok 3 third test\n')
255 result = subunit.TAP2SubUnit(self.tap, self.subunit)
256 self.assertEqual(0, result)
258 'test test 1 first test',
259 'success test 1 first test',
262 'test missing from TAP output',
264 'test test 3 third test',
265 'failure test 3 third test',
267 self.subunit.getvalue().splitlines())
269 def test_missing_test_no_plan_adds_error(self):
272 # not ok 3 third test
273 # results in three tests, with the second being created
274 self.tap.write('ok first test\n')
275 self.tap.write('not ok 3 third test\n')
277 result = subunit.TAP2SubUnit(self.tap, self.subunit)
278 self.assertEqual(0, result)
280 'test test 1 first test',
281 'success test 1 first test',
284 'test missing from TAP output',
286 'test test 3 third test',
287 'failure test 3 third test',
289 self.subunit.getvalue().splitlines())
291 def test_four_tests_in_a_row_trailing_plan(self):
293 # ok 1 - first test in a script with no plan at all
298 # results in four tests numbered and named
299 self.tap.write('ok 1 - first test in a script with trailing plan\n')
300 self.tap.write('not ok 2 - second\n')
301 self.tap.write('ok 3 - third\n')
302 self.tap.write('not ok 4 - fourth\n')
303 self.tap.write('1..4\n')
305 result = subunit.TAP2SubUnit(self.tap, self.subunit)
306 self.assertEqual(0, result)
308 'test test 1 - first test in a script with trailing plan',
309 'success test 1 - first test in a script with trailing plan',
310 'test test 2 - second',
311 'failure test 2 - second',
312 'test test 3 - third',
313 'success test 3 - third',
314 'test test 4 - fourth',
315 'failure test 4 - fourth'
317 self.subunit.getvalue().splitlines())
319 def test_four_tests_in_a_row_with_plan(self):
322 # ok 1 - first test in a script with no plan at all
326 # results in four tests numbered and named
327 self.tap.write('1..4\n')
328 self.tap.write('ok 1 - first test in a script with a plan\n')
329 self.tap.write('not ok 2 - second\n')
330 self.tap.write('ok 3 - third\n')
331 self.tap.write('not ok 4 - fourth\n')
333 result = subunit.TAP2SubUnit(self.tap, self.subunit)
334 self.assertEqual(0, result)
336 'test test 1 - first test in a script with a plan',
337 'success test 1 - first test in a script with a plan',
338 'test test 2 - second',
339 'failure test 2 - second',
340 'test test 3 - third',
341 'success test 3 - third',
342 'test test 4 - fourth',
343 'failure test 4 - fourth'
345 self.subunit.getvalue().splitlines())
347 def test_four_tests_in_a_row_no_plan(self):
349 # ok 1 - first test in a script with no plan at all
353 # results in four tests numbered and named
354 self.tap.write('ok 1 - first test in a script with no plan at all\n')
355 self.tap.write('not ok 2 - second\n')
356 self.tap.write('ok 3 - third\n')
357 self.tap.write('not ok 4 - fourth\n')
359 result = subunit.TAP2SubUnit(self.tap, self.subunit)
360 self.assertEqual(0, result)
362 'test test 1 - first test in a script with no plan at all',
363 'success test 1 - first test in a script with no plan at all',
364 'test test 2 - second',
365 'failure test 2 - second',
366 'test test 3 - third',
367 'success test 3 - third',
368 'test test 4 - fourth',
369 'failure test 4 - fourth'
371 self.subunit.getvalue().splitlines())
373 def test_todo_and_skip(self):
375 # not ok 1 - a fail but # TODO but is TODO
376 # not ok 2 - another fail # SKIP instead
377 # results in two tests, numbered and commented.
378 self.tap.write("not ok 1 - a fail but # TODO but is TODO\n")
379 self.tap.write("not ok 2 - another fail # SKIP instead\n")
381 result = subunit.TAP2SubUnit(self.tap, self.subunit)
382 self.assertEqual(0, result)
384 'test test 1 - a fail but',
385 'xfail test 1 - a fail but [',
388 'test test 2 - another fail',
389 'skip test 2 - another fail [',
393 self.subunit.getvalue().splitlines())
395 def test_leading_comments_add_to_next_test_log(self):
400 # results in a single test with the comment included
401 # in the first test and not the second.
402 self.tap.write("# comment\n")
403 self.tap.write("ok\n")
404 self.tap.write("ok\n")
406 result = subunit.TAP2SubUnit(self.tap, self.subunit)
407 self.assertEqual(0, result)
416 self.subunit.getvalue().splitlines())
418 def test_trailing_comments_are_included_in_last_test_log(self):
423 # results in a two tests, with the second having the comment
424 # attached to its log.
425 self.tap.write("ok\n")
426 self.tap.write("ok\n")
427 self.tap.write("# comment\n")
429 result = subunit.TAP2SubUnit(self.tap, self.subunit)
430 self.assertEqual(0, result)
439 self.subunit.getvalue().splitlines())
443 loader = subunit.tests.TestUtil.TestLoader()
444 result = loader.loadTestsFromName(__name__)