selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11 import subunithelper
  12
  13 class PlainFormatter(object):
  14
  15     def __init__(self, summaryfile, verbose, immediate, statistics,
  16             totaltests=None):
  17         self.verbose = verbose
  18         self.immediate = immediate
  19         self.statistics = statistics
  20         self.start_time = None
  21         self.test_output = {}
  22         self.suitesfailed = []
  23         self.suites_ok = 0
  24         self.skips = {}
  25         self.summaryfile = summaryfile
  26         self.index = 0
  27         self.name = None
  28         self.totalsuites = totaltests
  29
  30     def testsuite_count(self, count):
  31         self.totalsuites = count
  32
  33     def progress(self, offset, whence):
  34         pass
  35
  36     def report_time(self, time):
  37         if self.start_time is None:
  38             self.start_time = time
  39         self.last_time = time
  40
  41     def start_testsuite(self, name):
  42         self.index += 1
  43         self.name = name
  44         testsuite_start_time = self.last_time
  45
  46         duration = testsuite_start_time - self.start_time
  47
  48         if not self.verbose:
  49             self.test_output[name] = ""
  50
  51         out = "[%d" % self.index
  52         if self.totalsuites is not None:
  53             out += "/%d" % self.totalsuites
  54         out += " in %ds" % duration
  55         if self.suitesfailed:
  56             out += ", %d errors" % (len(self.suitesfailed),)
  57         out += "] %s" % name
  58         if self.immediate:
  59             sys.stdout.write(out + "\n")
  60         else:
  61             sys.stdout.write(out + ": ")
  62
  63     def output_msg(self, output):
  64         if self.verbose:
  65             sys.stdout.write(output)
  66         elif self.name is not None:
  67             self.test_output[self.name] += output
  68         else:
  69             sys.stdout.write(output)
  70
  71     def control_msg(self, output):
  72         #$self->output_msg($output)
  73         pass
  74
  75     def end_testsuite(self, name, result, reason):
  76         out = ""
  77         unexpected = 0
  78
  79         if not name in self.test_output:
  80             print "no output for name[%s]" % name
  81
  82         if result in ("success", "xfail"):
  83             self.suites_ok+=1
  84         else:
  85             self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason or ''))
  86             self.suitesfailed.append(name)
  87             if self.immediate and not self.verbose:
  88                 out += self.test_output[name]
  89             unexpected = 1
  90
  91         if not self.immediate:
  92             if not unexpected:
  93                 out += " ok\n"
  94             else:
  95                 out += " " + result.upper() + "\n"
  96
  97         sys.stdout.write(out)
  98
  99     def start_test(self, testname):
 100         pass
 101
 102     def end_test(self, testname, result, unexpected, reason):
 103         if not unexpected:
 104             self.test_output[self.name] = ""
 105             if not self.immediate:
 106                 sys.stdout.write({
 107                     'failure': 'f',
 108                     'xfail': 'X',
 109                     'skip': 's',
 110                     'success': '.'}.get(result, "?(%s)" % result))
 111             return
 112
 113         if reason is None:
 114             reason = ''
 115         reason = reason.strip()
 116
 117         self.test_output[self.name] += "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
 118
 119         if self.immediate and not self.verbose:
 120             print self.test_output[self.name]
 121             self.test_output[self.name] = ""
 122
 123         if not self.immediate:
 124             sys.stdout.write({
 125                'error': 'E',
 126                'failure': 'F',
 127                'success': 'S'}.get(result, "?"))
 128
 129     def summary(self):
 130         f = open(self.summaryfile, 'w+')
 131
 132         if self.suitesfailed:
 133             f.write("= Failed tests =\n")
 134
 135             for suite in self.suitesfailed:
 136                 f.write("== %s ==\n" % suite)
 137                 f.write(self.test_output[suite]+"\n\n")
 138
 139             f.write("\n")
 140
 141         if not self.immediate and not self.verbose:
 142             for suite in self.suitesfailed:
 143                 print "=" * 78
 144                 print "FAIL: %s" % suite
 145                 print self.test_output[suite]
 146                 print ""
 147
 148         f.write("= Skipped tests =\n")
 149         for reason in self.skips.keys():
 150             f.write(reason + "\n")
 151             for name in self.skips[reason]:
 152                 f.write("\t%s\n" % name)
 153             f.write("\n")
 154         f.close()
 155
 156         print "\nA summary with detailed information can be found in:"
 157         print "  %s" % self.summaryfile
 158
 159         if not self.suitesfailed:
 160             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 161                   self.statistics['TESTS_EXPECTED_FAIL'])
 162             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 163         else:
 164             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 165                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 166                 self.statistics['TESTS_ERROR'],
 167                 len(self.suitesfailed))
 168
 169     def skip_testsuite(self, name, reason="UNKNOWN"):
 170         self.skips.setdefault(reason, []).append(name)
 171         if self.totalsuites:
 172             self.totalsuites-=1
 173
 174 parser = optparse.OptionParser("format-subunit [options]")
 175 parser.add_option("--verbose", action="store_true",
 176     help="Be verbose")
 177 parser.add_option("--immediate", action="store_true",
 178     help="Show failures immediately, don't wait until test run has finished")
 179 parser.add_option("--prefix", type="string", default=".",
 180     help="Prefix to write summary to")
 181
 182 opts, args = parser.parse_args()
 183
 184 statistics = {
 185     'SUITES_FAIL': 0,
 186     'TESTS_UNEXPECTED_OK': 0,
 187     'TESTS_EXPECTED_OK': 0,
 188     'TESTS_UNEXPECTED_FAIL': 0,
 189     'TESTS_EXPECTED_FAIL': 0,
 190     'TESTS_ERROR': 0,
 191     'TESTS_SKIP': 0,
 192 }
 193
 194 def handle_sigint(sig, stack):
 195         sys.exit(0)
 196 signal.signal(signal.SIGINT, handle_sigint)
 197
 198 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 199     opts.immediate, statistics)
 200
 201 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 202
 203 msg_ops.summary()
 204
 205 sys.exit(expected_ret)