format-subunit: Improve formatting, simplify code.
[ira/wip.git] / selftest / format-subunit
1 #!/usr/bin/env python
2 # vim: expandtab
3 # Pretty-format subunit output
4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
5 # Published under the GNU GPL, v3 or later
6
7 import optparse
8 import os
9 import sys
10
11 import subunithelper
12
13 class PlainFormatter(object):
14
15     def __init__(self, summaryfile, verbose, immediate, statistics,
16             totaltests=None):
17         self.verbose = verbose
18         self.immediate = immediate
19         self.statistics = statistics
20         self.start_time = None
21         self.test_output = {}
22         self.suitesfailed = []
23         self.suites_ok = 0
24         self.skips = {}
25         self.summaryfile = summaryfile
26         self.index = 0
27         self.name = None
28         self.totalsuites = totaltests
29
30     def testsuite_count(self, count):
31         self.totalsuites = count
32
33     def report_time(self, time):
34         if self.start_time is None:
35             self.start_time = time
36         self.last_time = time
37
38     def start_testsuite(self, name):
39         self.index += 1
40         self.name = name
41         testsuite_start_time = self.last_time
42
43         duration = testsuite_start_time - self.start_time
44
45         if not self.verbose:
46             self.test_output[name] = "" 
47
48         out = "[%d" % self.index
49         if self.totalsuites is not None:
50             out += "/%d" % self.totalsuites
51         out += " in %ds" % duration
52         if self.suitesfailed:
53             out += ", %d errors" % (len(self.suitesfailed),)
54         out += "] %s" % name 
55         if self.immediate:
56             sys.stdout.write(out + "\n")
57         else:
58             sys.stdout.write(out + ": ")
59
60     def output_msg(self, output):
61         if self.verbose:
62             sys.stdout.write(output)
63         elif self.name is not None:
64             self.test_output[self.name] += output
65         else:
66             sys.stdout.write(output)
67
68     def control_msg(self, output):
69         #$self->output_msg($output)
70         pass
71
72     def end_testsuite(self, name, result, reason):
73         out = ""
74         unexpected = 0
75
76         if not name in self.test_output:
77             print "no output for name[%s]" % name
78
79         if result in ("success", "xfail"):
80             self.suites_ok+=1
81         else:
82             self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason))
83             self.suitesfailed.append(name)
84             if self.immediate and not self.verbose:
85                 out += self.test_output[name]
86             unexpected = 1
87
88         if not self.immediate:
89             if not unexpected:
90                 out += " ok\n"
91             else:
92                 out += " " + result.upper() + "\n"
93
94         sys.stdout.write(out)
95
96     def start_test(self, testname):
97         pass
98
99     def end_test(self, testname, result, unexpected, reason):
100         if not unexpected:
101             self.test_output[self.name] = ""
102             if not self.immediate:
103                 sys.stdout.write({
104                     'failure': 'f',
105                     'xfail': 'X',
106                     'skip': 's',
107                     'success': '.'}.get(result, "?(%s)" % result))
108             return
109
110         reason = reason.strip()
111
112         self.test_output[self.name] += "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
113
114         if self.immediate and not self.verbose:
115             print self.test_output[self.name]
116             self.test_output[self.name] = ""
117
118         if not self.immediate:
119             sys.stdout.write({
120                'error': 'E',
121                'failure': 'F',
122                'success': 'S'}.get(result, "?"))
123
124     def summary(self):
125         f = open(self.summaryfile, 'w+')
126
127         if self.suitesfailed:
128             f.write("= Failed tests =\n")
129
130             for suite in self.suitesfailed:
131                 f.write("== %s ==\n" % suite)
132                 f.write(self.test_output[suite]+"\n\n")
133
134             f.write("\n")
135
136         if not self.immediate and not self.verbose:
137             for suite in self.suitesfailed:
138                 print "=" * 78
139                 print "FAIL: %s" % suite
140                 print self.test_output[suite]
141                 print ""
142
143         f.write("= Skipped tests =\n")
144         for reason in self.skips.keys():
145             f.write(reason + "\n")
146             for name in self.skips[reason]:
147                 f.write("\t%s\n" % name)
148             f.write("\n")
149         f.close()
150
151         print "\nA summary with detailed information can be found in:"
152         print "  %s" % self.summaryfile
153
154         if not self.suitesfailed:
155             ok = (self.statistics['TESTS_EXPECTED_OK'] +
156                   self.statistics['TESTS_EXPECTED_FAIL'])
157             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
158         else:
159             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
160                 self.statistics['TESTS_UNEXPECTED_FAIL'],
161                 self.statistics['TESTS_ERROR'],
162                 len(self.suitesfailed))
163
164     def skip_testsuite(self, name, reason="UNKNOWN"):
165         self.skips.setdefault(reason, []).append(name)
166         if self.totalsuites:
167             self.totalsuites-=1
168
169 parser = optparse.OptionParser("format-subunit [options]")
170 parser.add_option("--verbose", action="store_true",
171     help="Be verbose")
172 parser.add_option("--immediate", action="store_true", 
173     help="Show failures immediately, don't wait until test run has finished")
174 parser.add_option("--prefix", type="string", default=".",
175     help="Prefix to write summary to")
176
177 opts, args = parser.parse_args()
178
179 statistics = {
180     'SUITES_FAIL': 0,
181     'TESTS_UNEXPECTED_OK': 0,
182     'TESTS_EXPECTED_OK': 0,
183     'TESTS_UNEXPECTED_FAIL': 0,
184     'TESTS_EXPECTED_FAIL': 0,
185     'TESTS_ERROR': 0,
186     'TESTS_SKIP': 0,
187 }
188
189 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
190     opts.immediate, statistics)
191
192 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
193
194 msg_ops.summary()
195
196 sys.exit(expected_ret)