selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11 import subunithelper
  12 import subunit
  13
  14 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
  15 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
  16
  17 class PlainFormatter(object):
  18
  19     def __init__(self, summaryfile, verbose, immediate, statistics,
  20             totaltests=None):
  21         self.verbose = verbose
  22         self.immediate = immediate
  23         self.statistics = statistics
  24         self.start_time = None
  25         self.test_output = {}
  26         self.suitesfailed = []
  27         self.suites_ok = 0
  28         self.skips = {}
  29         self.summaryfile = summaryfile
  30         self.index = 0
  31         self.name = None
  32         self._progress_level = 0
  33         self.totalsuites = totaltests
  34
  35     def progress(self, offset, whence):
  36         if whence == subunit.PROGRESS_POP:
  37             self._progress_level -= 1
  38         elif whence == subunit.PROGRESS_PUSH:
  39             self._progress_level += 1
  40         elif whence == subunit.PROGRESS_SET:
  41             if self._progress_level == 0:
  42                 self.totalsuites = offset
  43         elif whence == subunit.PROGRESS_CUR:
  44             raise NotImplementedError
  45
  46     def report_time(self, time):
  47         if self.start_time is None:
  48             self.start_time = time
  49         self.last_time = time
  50
  51     def start_testsuite(self, name):
  52         self.index += 1
  53         self.name = name
  54         testsuite_start_time = self.last_time
  55
  56         duration = testsuite_start_time - self.start_time
  57
  58         if not self.verbose:
  59             self.test_output[name] = ""
  60
  61         out = "[%d" % self.index
  62         if self.totalsuites is not None:
  63             out += "/%d" % self.totalsuites
  64         out += " in %ds" % duration
  65         if self.suitesfailed:
  66             out += ", %d errors" % (len(self.suitesfailed),)
  67         out += "] %s" % name
  68         if self.immediate:
  69             sys.stdout.write(out + "\n")
  70         else:
  71             sys.stdout.write(out + ": ")
  72
  73     def output_msg(self, output):
  74         if self.verbose:
  75             sys.stdout.write(output)
  76         elif self.name is not None:
  77             self.test_output[self.name] += output
  78         else:
  79             sys.stdout.write(output)
  80
  81     def control_msg(self, output):
  82         #$self->output_msg($output)
  83         pass
  84
  85     def end_testsuite(self, name, result, reason):
  86         out = ""
  87         unexpected = False
  88
  89         if not name in self.test_output:
  90             print "no output for name[%s]" % name
  91
  92         if result in ("success", "xfail"):
  93             self.suites_ok+=1
  94         else:
  95             self.output_msg("ERROR: Testsuite[%s]\n" % name)
  96             if reason is not None:
  97                 self.output_msg("REASON: %s\n" % (reason,))
  98             self.suitesfailed.append(name)
  99             if self.immediate and not self.verbose:
 100                 out += self.test_output[name]
 101             unexpected = True
 102
 103         if not self.immediate:
 104             if not unexpected:
 105                 out += " ok\n"
 106             else:
 107                 out += " " + result.upper() + "\n"
 108
 109         sys.stdout.write(out)
 110
 111     def start_test(self, testname):
 112         pass
 113
 114     def end_test(self, testname, result, unexpected, reason=None):
 115         if not unexpected:
 116             self.test_output[self.name] = ""
 117             if not self.immediate:
 118                 sys.stdout.write({
 119                     'failure': 'f',
 120                     'xfail': 'X',
 121                     'skip': 's',
 122                     'success': '.'}.get(result, "?(%s)" % result))
 123             return
 124
 125         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 126         if reason is not None:
 127             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
 128
 129         if self.immediate and not self.verbose:
 130             print self.test_output[self.name]
 131             self.test_output[self.name] = ""
 132
 133         if not self.immediate:
 134             sys.stdout.write({
 135                'error': 'E',
 136                'failure': 'F',
 137                'success': 'S'}.get(result, "?"))
 138
 139     def summary(self):
 140         f = open(self.summaryfile, 'w+')
 141
 142         if self.suitesfailed:
 143             f.write("= Failed tests =\n")
 144
 145             for suite in self.suitesfailed:
 146                 f.write("== %s ==\n" % suite)
 147                 f.write(self.test_output[suite]+"\n\n")
 148
 149             f.write("\n")
 150
 151         if not self.immediate and not self.verbose:
 152             for suite in self.suitesfailed:
 153                 print "=" * 78
 154                 print "FAIL: %s" % suite
 155                 print self.test_output[suite]
 156                 print ""
 157
 158         f.write("= Skipped tests =\n")
 159         for reason in self.skips.keys():
 160             f.write(reason + "\n")
 161             for name in self.skips[reason]:
 162                 f.write("\t%s\n" % name)
 163             f.write("\n")
 164         f.close()
 165
 166         print "\nA summary with detailed information can be found in:"
 167         print "  %s" % self.summaryfile
 168
 169         if not self.suitesfailed:
 170             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 171                   self.statistics['TESTS_EXPECTED_FAIL'])
 172             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 173         else:
 174             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 175                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 176                 self.statistics['TESTS_ERROR'],
 177                 len(self.suitesfailed))
 178
 179     def skip_testsuite(self, name, reason="UNKNOWN"):
 180         self.skips.setdefault(reason, []).append(name)
 181         if self.totalsuites:
 182             self.totalsuites-=1
 183
 184 parser = optparse.OptionParser("format-subunit [options]")
 185 parser.add_option("--verbose", action="store_true",
 186     help="Be verbose")
 187 parser.add_option("--immediate", action="store_true",
 188     help="Show failures immediately, don't wait until test run has finished")
 189 parser.add_option("--prefix", type="string", default=".",
 190     help="Prefix to write summary to")
 191
 192 opts, args = parser.parse_args()
 193
 194 statistics = {
 195     'SUITES_FAIL': 0,
 196     'TESTS_UNEXPECTED_OK': 0,
 197     'TESTS_EXPECTED_OK': 0,
 198     'TESTS_UNEXPECTED_FAIL': 0,
 199     'TESTS_EXPECTED_FAIL': 0,
 200     'TESTS_ERROR': 0,
 201     'TESTS_SKIP': 0,
 202 }
 203
 204 def handle_sigint(sig, stack):
 205         sys.exit(0)
 206 signal.signal(signal.SIGINT, handle_sigint)
 207
 208 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 209     opts.immediate, statistics)
 210
 211 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 212
 213 msg_ops.summary()
 214
 215 sys.exit(expected_ret)