43cdf096f2fbf468e3841a4c6f5f3b1646cdf9ca
[ira/wip.git] / selftest / format-subunit
1 #!/usr/bin/env python
2 # vim: expandtab
3 # Pretty-format subunit output
4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
5 # Published under the GNU GPL, v3 or later
6
7 import optparse
8 import os
9 import signal
10 import sys
11
12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
14
15 import subunithelper
16 import subunit
17
18
19 def format_time(delta):
20     minutes, seconds = divmod(delta.seconds, 60)
21     hours, minutes = divmod(minutes, 60)
22     ret = ""
23     if hours:
24         ret += "%dh" % hours
25     if minutes:
26         ret += "%dm" % minutes
27     ret += "%ds" % seconds
28     return ret
29
30
31 class PlainFormatter(subunithelper.TestsuiteEnabledTestResult):
32
33     def __init__(self, summaryfile, verbose, immediate, statistics,
34             totaltests=None):
35         self.verbose = verbose
36         self.immediate = immediate
37         self.statistics = statistics
38         self.start_time = None
39         self.test_output = {}
40         self.suitesfailed = []
41         self.suites_ok = 0
42         self.skips = {}
43         self.summaryfile = summaryfile
44         self.index = 0
45         self.name = None
46         self._progress_level = 0
47         self.totalsuites = totaltests
48         self.last_time = None
49
50     def progress(self, offset, whence):
51         if whence == subunit.PROGRESS_POP:
52             self._progress_level -= 1
53         elif whence == subunit.PROGRESS_PUSH:
54             self._progress_level += 1
55         elif whence == subunit.PROGRESS_SET:
56             if self._progress_level == 0:
57                 self.totalsuites = offset
58         elif whence == subunit.PROGRESS_CUR:
59             raise NotImplementedError
60
61     def time(self, dt):
62         if self.start_time is None:
63             self.start_time = dt
64         self.last_time = dt
65
66     def start_testsuite(self, name):
67         self.index += 1
68         self.name = name
69
70         if not self.verbose:
71             self.test_output[name] = ""
72
73         out = "[%d" % self.index
74         if self.totalsuites is not None:
75             out += "/%d" % self.totalsuites
76         if self.start_time is not None:
77             out += " in " + format_time(self.last_time - self.start_time)
78         if self.suitesfailed:
79             out += ", %d errors" % (len(self.suitesfailed),)
80         out += "] %s" % name
81         if self.immediate:
82             sys.stdout.write(out + "\n")
83         else:
84             sys.stdout.write(out + ": ")
85
86     def output_msg(self, output):
87         if self.verbose:
88             sys.stdout.write(output)
89         elif self.name is not None:
90             self.test_output[self.name] += output
91         else:
92             sys.stdout.write(output)
93
94     def control_msg(self, output):
95         #$self->output_msg($output)
96         pass
97
98     def end_testsuite(self, name, result, reason):
99         out = ""
100         unexpected = False
101
102         if not name in self.test_output:
103             print "no output for name[%s]" % name
104
105         if result in ("success", "xfail"):
106             self.suites_ok+=1
107         else:
108             self.output_msg("ERROR: Testsuite[%s]\n" % name)
109             if reason is not None:
110                 self.output_msg("REASON: %s\n" % (reason,))
111             self.suitesfailed.append(name)
112             if self.immediate and not self.verbose and name in self.test_output:
113                 out += self.test_output[name]
114             unexpected = True
115
116         if not self.immediate:
117             if not unexpected:
118                 out += " ok\n"
119             else:
120                 out += " " + result.upper() + "\n"
121
122         sys.stdout.write(out)
123
124     def startTest(self, test):
125         pass
126
127     def addSuccess(self, test, details=None):
128         self.end_test(test.id(), "success", False, details)
129
130     def addError(self, test, details=None):
131         self.end_test(test.id(), "error", True, details)
132
133     def addFailure(self, test, details=None):
134         self.end_test(test.id(), "failure", True, details)
135
136     def addSkip(self, test, details=None):
137         self.end_test(test.id(), "skip", False, details)
138
139     def addExpectedFail(self, test, details=None):
140         self.end_test(test.id(), "xfail", False, details)
141
142     def end_test(self, testname, result, unexpected, reason=None):
143         if not unexpected:
144             self.test_output[self.name] = ""
145             if not self.immediate:
146                 sys.stdout.write({
147                     'failure': 'f',
148                     'xfail': 'X',
149                     'skip': 's',
150                     'success': '.'}.get(result, "?(%s)" % result))
151             return
152
153         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
154         if reason is not None:
155             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
156
157         if self.immediate and not self.verbose:
158             print self.test_output[self.name]
159             self.test_output[self.name] = ""
160
161         if not self.immediate:
162             sys.stdout.write({
163                'error': 'E',
164                'failure': 'F',
165                'success': 'S'}.get(result, "?"))
166
167     def summary(self):
168         f = open(self.summaryfile, 'w+')
169
170         if self.suitesfailed:
171             f.write("= Failed tests =\n")
172
173             for suite in self.suitesfailed:
174                 f.write("== %s ==\n" % suite)
175                 if suite in self.test_output:
176                     f.write(self.test_output[suite]+"\n\n")
177
178             f.write("\n")
179
180         if not self.immediate and not self.verbose:
181             for suite in self.suitesfailed:
182                 print "=" * 78
183                 print "FAIL: %s" % suite
184                 if suite in self.test_output:
185                     print self.test_output[suite]
186                 print ""
187
188         f.write("= Skipped tests =\n")
189         for reason in self.skips.keys():
190             f.write(reason + "\n")
191             for name in self.skips[reason]:
192                 f.write("\t%s\n" % name)
193             f.write("\n")
194         f.close()
195
196         print "\nA summary with detailed information can be found in:"
197         print "  %s" % self.summaryfile
198
199         if not self.suitesfailed:
200             ok = (self.statistics['TESTS_EXPECTED_OK'] +
201                   self.statistics['TESTS_EXPECTED_FAIL'])
202             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
203         else:
204             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
205                 self.statistics['TESTS_UNEXPECTED_FAIL'],
206                 self.statistics['TESTS_ERROR'],
207                 len(self.suitesfailed))
208
209     def skip_testsuite(self, name, reason="UNKNOWN"):
210         self.skips.setdefault(reason, []).append(name)
211         if self.totalsuites:
212             self.totalsuites-=1
213
214 parser = optparse.OptionParser("format-subunit [options]")
215 parser.add_option("--verbose", action="store_true",
216     help="Be verbose")
217 parser.add_option("--immediate", action="store_true", 
218     help="Show failures immediately, don't wait until test run has finished")
219 parser.add_option("--prefix", type="string", default=".",
220     help="Prefix to write summary to")
221
222 opts, args = parser.parse_args()
223
224 statistics = {
225     'SUITES_FAIL': 0,
226     'TESTS_UNEXPECTED_OK': 0,
227     'TESTS_EXPECTED_OK': 0,
228     'TESTS_UNEXPECTED_FAIL': 0,
229     'TESTS_EXPECTED_FAIL': 0,
230     'TESTS_ERROR': 0,
231     'TESTS_SKIP': 0,
232 }
233
234 def handle_sigint(sig, stack):
235         sys.exit(0)
236 signal.signal(signal.SIGINT, handle_sigint)
237
238 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
239     opts.immediate, statistics)
240
241 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
242
243 msg_ops.summary()
244
245 sys.exit(expected_ret)