selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11 import subunithelper
  12 import subunit
  13
  14 class PlainFormatter(object):
  15
  16     def __init__(self, summaryfile, verbose, immediate, statistics,
  17             totaltests=None):
  18         self.verbose = verbose
  19         self.immediate = immediate
  20         self.statistics = statistics
  21         self.start_time = None
  22         self.test_output = {}
  23         self.suitesfailed = []
  24         self.suites_ok = 0
  25         self.skips = {}
  26         self.summaryfile = summaryfile
  27         self.index = 0
  28         self.name = None
  29         self._progress_level = 0
  30         self.totalsuites = totaltests
  31
  32     def progress(self, offset, whence):
  33         if whence == subunit.PROGRESS_POP:
  34             self._progress_level -= 1
  35         elif whence == subunit.PROGRESS_PUSH:
  36             self._progress_level += 1
  37         elif whence == subunit.PROGRESS_SET:
  38             if self._progress_level == 0:
  39                 self.totalsuites = offset
  40         elif whence == subunit.PROGRESS_CUR:
  41             raise NotImplementedError
  42
  43     def report_time(self, time):
  44         if self.start_time is None:
  45             self.start_time = time
  46         self.last_time = time
  47
  48     def start_testsuite(self, name):
  49         self.index += 1
  50         self.name = name
  51         testsuite_start_time = self.last_time
  52
  53         duration = testsuite_start_time - self.start_time
  54
  55         if not self.verbose:
  56             self.test_output[name] = ""
  57
  58         out = "[%d" % self.index
  59         if self.totalsuites is not None:
  60             out += "/%d" % self.totalsuites
  61         out += " in %ds" % duration
  62         if self.suitesfailed:
  63             out += ", %d errors" % (len(self.suitesfailed),)
  64         out += "] %s" % name
  65         if self.immediate:
  66             sys.stdout.write(out + "\n")
  67         else:
  68             sys.stdout.write(out + ": ")
  69
  70     def output_msg(self, output):
  71         if self.verbose:
  72             sys.stdout.write(output)
  73         elif self.name is not None:
  74             self.test_output[self.name] += output
  75         else:
  76             sys.stdout.write(output)
  77
  78     def control_msg(self, output):
  79         #$self->output_msg($output)
  80         pass
  81
  82     def end_testsuite(self, name, result, reason):
  83         out = ""
  84         unexpected = False
  85
  86         if not name in self.test_output:
  87             print "no output for name[%s]" % name
  88
  89         if result in ("success", "xfail"):
  90             self.suites_ok+=1
  91         else:
  92             self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason or ''))
  93             self.suitesfailed.append(name)
  94             if self.immediate and not self.verbose:
  95                 out += self.test_output[name]
  96             unexpected = True
  97
  98         if not self.immediate:
  99             if not unexpected:
 100                 out += " ok\n"
 101             else:
 102                 out += " " + result.upper() + "\n"
 103
 104         sys.stdout.write(out)
 105
 106     def start_test(self, testname):
 107         pass
 108
 109     def end_test(self, testname, result, unexpected, reason):
 110         if not unexpected:
 111             self.test_output[self.name] = ""
 112             if not self.immediate:
 113                 sys.stdout.write({
 114                     'failure': 'f',
 115                     'xfail': 'X',
 116                     'skip': 's',
 117                     'success': '.'}.get(result, "?(%s)" % result))
 118             return
 119
 120         if reason is None:
 121             reason = ''
 122         reason = reason.strip()
 123
 124         self.test_output[self.name] += "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
 125
 126         if self.immediate and not self.verbose:
 127             print self.test_output[self.name]
 128             self.test_output[self.name] = ""
 129
 130         if not self.immediate:
 131             sys.stdout.write({
 132                'error': 'E',
 133                'failure': 'F',
 134                'success': 'S'}.get(result, "?"))
 135
 136     def summary(self):
 137         f = open(self.summaryfile, 'w+')
 138
 139         if self.suitesfailed:
 140             f.write("= Failed tests =\n")
 141
 142             for suite in self.suitesfailed:
 143                 f.write("== %s ==\n" % suite)
 144                 f.write(self.test_output[suite]+"\n\n")
 145
 146             f.write("\n")
 147
 148         if not self.immediate and not self.verbose:
 149             for suite in self.suitesfailed:
 150                 print "=" * 78
 151                 print "FAIL: %s" % suite
 152                 print self.test_output[suite]
 153                 print ""
 154
 155         f.write("= Skipped tests =\n")
 156         for reason in self.skips.keys():
 157             f.write(reason + "\n")
 158             for name in self.skips[reason]:
 159                 f.write("\t%s\n" % name)
 160             f.write("\n")
 161         f.close()
 162
 163         print "\nA summary with detailed information can be found in:"
 164         print "  %s" % self.summaryfile
 165
 166         if not self.suitesfailed:
 167             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 168                   self.statistics['TESTS_EXPECTED_FAIL'])
 169             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 170         else:
 171             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 172                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 173                 self.statistics['TESTS_ERROR'],
 174                 len(self.suitesfailed))
 175
 176     def skip_testsuite(self, name, reason="UNKNOWN"):
 177         self.skips.setdefault(reason, []).append(name)
 178         if self.totalsuites:
 179             self.totalsuites-=1
 180
 181 parser = optparse.OptionParser("format-subunit [options]")
 182 parser.add_option("--verbose", action="store_true",
 183     help="Be verbose")
 184 parser.add_option("--immediate", action="store_true",
 185     help="Show failures immediately, don't wait until test run has finished")
 186 parser.add_option("--prefix", type="string", default=".",
 187     help="Prefix to write summary to")
 188
 189 opts, args = parser.parse_args()
 190
 191 statistics = {
 192     'SUITES_FAIL': 0,
 193     'TESTS_UNEXPECTED_OK': 0,
 194     'TESTS_EXPECTED_OK': 0,
 195     'TESTS_UNEXPECTED_FAIL': 0,
 196     'TESTS_EXPECTED_FAIL': 0,
 197     'TESTS_ERROR': 0,
 198     'TESTS_SKIP': 0,
 199 }
 200
 201 def handle_sigint(sig, stack):
 202         sys.exit(0)
 203 signal.signal(signal.SIGINT, handle_sigint)
 204
 205 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 206     opts.immediate, statistics)
 207
 208 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 209
 210 msg_ops.summary()
 211
 212 sys.exit(expected_ret)