6d2ceb10478cf623df727bcb74a2b930e82db19a
[ira/wip.git] / selftest / format-subunit
1 #!/usr/bin/env python
2 # vim: expandtab
3 # Pretty-format subunit output
4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
5 # Published under the GNU GPL, v3 or later
6
7 import optparse
8 import os
9 import signal
10 import sys
11 import subunithelper
12 import subunit
13
14 class PlainFormatter(object):
15
16     def __init__(self, summaryfile, verbose, immediate, statistics,
17             totaltests=None):
18         self.verbose = verbose
19         self.immediate = immediate
20         self.statistics = statistics
21         self.start_time = None
22         self.test_output = {}
23         self.suitesfailed = []
24         self.suites_ok = 0
25         self.skips = {}
26         self.summaryfile = summaryfile
27         self.index = 0
28         self.name = None
29         self._progress_level = 0
30         self.totalsuites = totaltests
31
32     def progress(self, offset, whence):
33         if whence == subunit.PROGRESS_POP:
34             self._progress_level -= 1
35         elif whence == subunit.PROGRESS_PUSH:
36             self._progress_level += 1
37         elif whence == subunit.PROGRESS_SET:
38             if self._progress_level == 0:
39                 self.totalsuites = offset
40         elif whence == subunit.PROGRESS_CUR:
41             raise NotImplementedError
42
43     def report_time(self, time):
44         if self.start_time is None:
45             self.start_time = time
46         self.last_time = time
47
48     def start_testsuite(self, name):
49         self.index += 1
50         self.name = name
51         testsuite_start_time = self.last_time
52
53         duration = testsuite_start_time - self.start_time
54
55         if not self.verbose:
56             self.test_output[name] = "" 
57
58         out = "[%d" % self.index
59         if self.totalsuites is not None:
60             out += "/%d" % self.totalsuites
61         out += " in %ds" % duration
62         if self.suitesfailed:
63             out += ", %d errors" % (len(self.suitesfailed),)
64         out += "] %s" % name 
65         if self.immediate:
66             sys.stdout.write(out + "\n")
67         else:
68             sys.stdout.write(out + ": ")
69
70     def output_msg(self, output):
71         if self.verbose:
72             sys.stdout.write(output)
73         elif self.name is not None:
74             self.test_output[self.name] += output
75         else:
76             sys.stdout.write(output)
77
78     def control_msg(self, output):
79         #$self->output_msg($output)
80         pass
81
82     def end_testsuite(self, name, result, reason):
83         out = ""
84         unexpected = 0
85
86         if not name in self.test_output:
87             print "no output for name[%s]" % name
88
89         if result in ("success", "xfail"):
90             self.suites_ok+=1
91         else:
92             self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason or ''))
93             self.suitesfailed.append(name)
94             if self.immediate and not self.verbose:
95                 out += self.test_output[name]
96             unexpected = 1
97
98         if not self.immediate:
99             if not unexpected:
100                 out += " ok\n"
101             else:
102                 out += " " + result.upper() + "\n"
103
104         sys.stdout.write(out)
105
106     def start_test(self, testname):
107         pass
108
109     def end_test(self, testname, result, unexpected, reason):
110         if not unexpected:
111             self.test_output[self.name] = ""
112             if not self.immediate:
113                 sys.stdout.write({
114                     'failure': 'f',
115                     'xfail': 'X',
116                     'skip': 's',
117                     'success': '.'}.get(result, "?(%s)" % result))
118             return
119
120         if reason is None:
121             reason = ''
122         reason = reason.strip()
123
124         self.test_output[self.name] += "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
125
126         if self.immediate and not self.verbose:
127             print self.test_output[self.name]
128             self.test_output[self.name] = ""
129
130         if not self.immediate:
131             sys.stdout.write({
132                'error': 'E',
133                'failure': 'F',
134                'success': 'S'}.get(result, "?"))
135
136     def summary(self):
137         f = open(self.summaryfile, 'w+')
138
139         if self.suitesfailed:
140             f.write("= Failed tests =\n")
141
142             for suite in self.suitesfailed:
143                 f.write("== %s ==\n" % suite)
144                 f.write(self.test_output[suite]+"\n\n")
145
146             f.write("\n")
147
148         if not self.immediate and not self.verbose:
149             for suite in self.suitesfailed:
150                 print "=" * 78
151                 print "FAIL: %s" % suite
152                 print self.test_output[suite]
153                 print ""
154
155         f.write("= Skipped tests =\n")
156         for reason in self.skips.keys():
157             f.write(reason + "\n")
158             for name in self.skips[reason]:
159                 f.write("\t%s\n" % name)
160             f.write("\n")
161         f.close()
162
163         print "\nA summary with detailed information can be found in:"
164         print "  %s" % self.summaryfile
165
166         if not self.suitesfailed:
167             ok = (self.statistics['TESTS_EXPECTED_OK'] +
168                   self.statistics['TESTS_EXPECTED_FAIL'])
169             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
170         else:
171             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
172                 self.statistics['TESTS_UNEXPECTED_FAIL'],
173                 self.statistics['TESTS_ERROR'],
174                 len(self.suitesfailed))
175
176     def skip_testsuite(self, name, reason="UNKNOWN"):
177         self.skips.setdefault(reason, []).append(name)
178         if self.totalsuites:
179             self.totalsuites-=1
180
181 parser = optparse.OptionParser("format-subunit [options]")
182 parser.add_option("--verbose", action="store_true",
183     help="Be verbose")
184 parser.add_option("--immediate", action="store_true", 
185     help="Show failures immediately, don't wait until test run has finished")
186 parser.add_option("--prefix", type="string", default=".",
187     help="Prefix to write summary to")
188
189 opts, args = parser.parse_args()
190
191 statistics = {
192     'SUITES_FAIL': 0,
193     'TESTS_UNEXPECTED_OK': 0,
194     'TESTS_EXPECTED_OK': 0,
195     'TESTS_UNEXPECTED_FAIL': 0,
196     'TESTS_EXPECTED_FAIL': 0,
197     'TESTS_ERROR': 0,
198     'TESTS_SKIP': 0,
199 }
200
201 def handle_sigint(sig, stack):
202         sys.exit(0)
203 signal.signal(signal.SIGINT, handle_sigint)
204
205 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
206     opts.immediate, statistics)
207
208 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
209
210 msg_ops.summary()
211
212 sys.exit(expected_ret)