subunit: Use standard subunit test protocol client, use standard name for startTest.
[ira/wip.git] / selftest / format-subunit
1 #!/usr/bin/env python
2 # vim: expandtab
3 # Pretty-format subunit output
4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
5 # Published under the GNU GPL, v3 or later
6
7 import optparse
8 import os
9 import signal
10 import sys
11
12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
14
15 import subunithelper
16 import subunit
17 import testtools
18
19 def format_time(t):
20     minutes, seconds = divmod(t, 60)
21     hours, minutes = divmod(minutes, 60)
22     ret = ""
23     if hours:
24         ret += "%dh" % hours
25     if minutes:
26         ret += "%dm" % minutes
27     ret += "%ds" % seconds
28     return ret
29
30
31 class PlainFormatter(subunithelper.TestsuiteEnabledTestResult):
32
33     def __init__(self, summaryfile, verbose, immediate, statistics,
34             totaltests=None):
35         self.verbose = verbose
36         self.immediate = immediate
37         self.statistics = statistics
38         self.start_time = None
39         self.test_output = {}
40         self.suitesfailed = []
41         self.suites_ok = 0
42         self.skips = {}
43         self.summaryfile = summaryfile
44         self.index = 0
45         self.name = None
46         self._progress_level = 0
47         self.totalsuites = totaltests
48
49     def progress(self, offset, whence):
50         if whence == subunit.PROGRESS_POP:
51             self._progress_level -= 1
52         elif whence == subunit.PROGRESS_PUSH:
53             self._progress_level += 1
54         elif whence == subunit.PROGRESS_SET:
55             if self._progress_level == 0:
56                 self.totalsuites = offset
57         elif whence == subunit.PROGRESS_CUR:
58             raise NotImplementedError
59
60     def report_time(self, time):
61         if self.start_time is None:
62             self.start_time = time
63         self.last_time = time
64
65     def start_testsuite(self, name):
66         self.index += 1
67         self.name = name
68         testsuite_start_time = self.last_time
69
70         duration = testsuite_start_time - self.start_time
71
72         if not self.verbose:
73             self.test_output[name] = ""
74
75         out = "[%d" % self.index
76         if self.totalsuites is not None:
77             out += "/%d" % self.totalsuites
78         out += " in " + format_time(duration)
79         if self.suitesfailed:
80             out += ", %d errors" % (len(self.suitesfailed),)
81         out += "] %s" % name
82         if self.immediate:
83             sys.stdout.write(out + "\n")
84         else:
85             sys.stdout.write(out + ": ")
86
87     def output_msg(self, output):
88         if self.verbose:
89             sys.stdout.write(output)
90         elif self.name is not None:
91             self.test_output[self.name] += output
92         else:
93             sys.stdout.write(output)
94
95     def control_msg(self, output):
96         #$self->output_msg($output)
97         pass
98
99     def end_testsuite(self, name, result, reason):
100         out = ""
101         unexpected = False
102
103         if not name in self.test_output:
104             print "no output for name[%s]" % name
105
106         if result in ("success", "xfail"):
107             self.suites_ok+=1
108         else:
109             self.output_msg("ERROR: Testsuite[%s]\n" % name)
110             if reason is not None:
111                 self.output_msg("REASON: %s\n" % (reason,))
112             self.suitesfailed.append(name)
113             if self.immediate and not self.verbose and name in self.test_output:
114                 out += self.test_output[name]
115             unexpected = True
116
117         if not self.immediate:
118             if not unexpected:
119                 out += " ok\n"
120             else:
121                 out += " " + result.upper() + "\n"
122
123         sys.stdout.write(out)
124
125     def startTest(self, testname):
126         pass
127
128     def end_test(self, testname, result, unexpected, reason=None):
129         if not unexpected:
130             self.test_output[self.name] = ""
131             if not self.immediate:
132                 sys.stdout.write({
133                     'failure': 'f',
134                     'xfail': 'X',
135                     'skip': 's',
136                     'success': '.'}.get(result, "?(%s)" % result))
137             return
138
139         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
140         if reason is not None:
141             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
142
143         if self.immediate and not self.verbose:
144             print self.test_output[self.name]
145             self.test_output[self.name] = ""
146
147         if not self.immediate:
148             sys.stdout.write({
149                'error': 'E',
150                'failure': 'F',
151                'success': 'S'}.get(result, "?"))
152
153     def summary(self):
154         f = open(self.summaryfile, 'w+')
155
156         if self.suitesfailed:
157             f.write("= Failed tests =\n")
158
159             for suite in self.suitesfailed:
160                 f.write("== %s ==\n" % suite)
161                 if suite in self.test_output:
162                     f.write(self.test_output[suite]+"\n\n")
163
164             f.write("\n")
165
166         if not self.immediate and not self.verbose:
167             for suite in self.suitesfailed:
168                 print "=" * 78
169                 print "FAIL: %s" % suite
170                 if suite in self.test_output:
171                     print self.test_output[suite]
172                 print ""
173
174         f.write("= Skipped tests =\n")
175         for reason in self.skips.keys():
176             f.write(reason + "\n")
177             for name in self.skips[reason]:
178                 f.write("\t%s\n" % name)
179             f.write("\n")
180         f.close()
181
182         print "\nA summary with detailed information can be found in:"
183         print "  %s" % self.summaryfile
184
185         if not self.suitesfailed:
186             ok = (self.statistics['TESTS_EXPECTED_OK'] +
187                   self.statistics['TESTS_EXPECTED_FAIL'])
188             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
189         else:
190             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
191                 self.statistics['TESTS_UNEXPECTED_FAIL'],
192                 self.statistics['TESTS_ERROR'],
193                 len(self.suitesfailed))
194
195     def skip_testsuite(self, name, reason="UNKNOWN"):
196         self.skips.setdefault(reason, []).append(name)
197         if self.totalsuites:
198             self.totalsuites-=1
199
200 parser = optparse.OptionParser("format-subunit [options]")
201 parser.add_option("--verbose", action="store_true",
202     help="Be verbose")
203 parser.add_option("--immediate", action="store_true", 
204     help="Show failures immediately, don't wait until test run has finished")
205 parser.add_option("--prefix", type="string", default=".",
206     help="Prefix to write summary to")
207
208 opts, args = parser.parse_args()
209
210 statistics = {
211     'SUITES_FAIL': 0,
212     'TESTS_UNEXPECTED_OK': 0,
213     'TESTS_EXPECTED_OK': 0,
214     'TESTS_UNEXPECTED_FAIL': 0,
215     'TESTS_EXPECTED_FAIL': 0,
216     'TESTS_ERROR': 0,
217     'TESTS_SKIP': 0,
218 }
219
220 def handle_sigint(sig, stack):
221         sys.exit(0)
222 signal.signal(signal.SIGINT, handle_sigint)
223
224 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
225     opts.immediate, statistics)
226
227 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
228
229 msg_ops.summary()
230
231 sys.exit(expected_ret)