selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import signal
  10 import sys
  11
  12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
  13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
  14
  15 import subunithelper
  16 import subunit
  17
  18
  19 def format_time(delta):
  20     minutes, seconds = divmod(delta.seconds, 60)
  21     hours, minutes = divmod(minutes, 60)
  22     ret = ""
  23     if hours:
  24         ret += "%dh" % hours
  25     if minutes:
  26         ret += "%dm" % minutes
  27     ret += "%ds" % seconds
  28     return ret
  29
  30
  31 class PlainFormatter(subunithelper.TestsuiteEnabledTestResult):
  32
  33     def __init__(self, summaryfile, verbose, immediate, statistics,
  34             totaltests=None):
  35         self.verbose = verbose
  36         self.immediate = immediate
  37         self.statistics = statistics
  38         self.start_time = None
  39         self.test_output = {}
  40         self.suitesfailed = []
  41         self.suites_ok = 0
  42         self.skips = {}
  43         self.summaryfile = summaryfile
  44         self.index = 0
  45         self.name = None
  46         self._progress_level = 0
  47         self.totalsuites = totaltests
  48
  49     def progress(self, offset, whence):
  50         if whence == subunit.PROGRESS_POP:
  51             self._progress_level -= 1
  52         elif whence == subunit.PROGRESS_PUSH:
  53             self._progress_level += 1
  54         elif whence == subunit.PROGRESS_SET:
  55             if self._progress_level == 0:
  56                 self.totalsuites = offset
  57         elif whence == subunit.PROGRESS_CUR:
  58             raise NotImplementedError
  59
  60     def time(self, dt):
  61         if self.start_time is None:
  62             self.start_time = dt
  63         self.last_time = dt
  64
  65     def start_testsuite(self, name):
  66         self.index += 1
  67         self.name = name
  68         testsuite_start_time = self.last_time
  69
  70         duration = testsuite_start_time - self.start_time
  71
  72         if not self.verbose:
  73             self.test_output[name] = ""
  74
  75         out = "[%d" % self.index
  76         if self.totalsuites is not None:
  77             out += "/%d" % self.totalsuites
  78         out += " in " + format_time(duration)
  79         if self.suitesfailed:
  80             out += ", %d errors" % (len(self.suitesfailed),)
  81         out += "] %s" % name
  82         if self.immediate:
  83             sys.stdout.write(out + "\n")
  84         else:
  85             sys.stdout.write(out + ": ")
  86
  87     def output_msg(self, output):
  88         if self.verbose:
  89             sys.stdout.write(output)
  90         elif self.name is not None:
  91             self.test_output[self.name] += output
  92         else:
  93             sys.stdout.write(output)
  94
  95     def control_msg(self, output):
  96         #$self->output_msg($output)
  97         pass
  98
  99     def end_testsuite(self, name, result, reason):
 100         out = ""
 101         unexpected = False
 102
 103         if not name in self.test_output:
 104             print "no output for name[%s]" % name
 105
 106         if result in ("success", "xfail"):
 107             self.suites_ok+=1
 108         else:
 109             self.output_msg("ERROR: Testsuite[%s]\n" % name)
 110             if reason is not None:
 111                 self.output_msg("REASON: %s\n" % (reason,))
 112             self.suitesfailed.append(name)
 113             if self.immediate and not self.verbose and name in self.test_output:
 114                 out += self.test_output[name]
 115             unexpected = True
 116
 117         if not self.immediate:
 118             if not unexpected:
 119                 out += " ok\n"
 120             else:
 121                 out += " " + result.upper() + "\n"
 122
 123         sys.stdout.write(out)
 124
 125     def startTest(self, test):
 126         pass
 127
 128     def end_test(self, testname, result, unexpected, reason=None):
 129         if not unexpected:
 130             self.test_output[self.name] = ""
 131             if not self.immediate:
 132                 sys.stdout.write({
 133                     'failure': 'f',
 134                     'xfail': 'X',
 135                     'skip': 's',
 136                     'success': '.'}.get(result, "?(%s)" % result))
 137             return
 138
 139         self.test_output[self.name] += "UNEXPECTED(%s): %s\n" % (result, testname)
 140         if reason is not None:
 141             self.test_output[self.name] += "REASON: %s\n" % (reason.strip(),)
 142
 143         if self.immediate and not self.verbose:
 144             print self.test_output[self.name]
 145             self.test_output[self.name] = ""
 146
 147         if not self.immediate:
 148             sys.stdout.write({
 149                'error': 'E',
 150                'failure': 'F',
 151                'success': 'S'}.get(result, "?"))
 152
 153     def summary(self):
 154         f = open(self.summaryfile, 'w+')
 155
 156         if self.suitesfailed:
 157             f.write("= Failed tests =\n")
 158
 159             for suite in self.suitesfailed:
 160                 f.write("== %s ==\n" % suite)
 161                 if suite in self.test_output:
 162                     f.write(self.test_output[suite]+"\n\n")
 163
 164             f.write("\n")
 165
 166         if not self.immediate and not self.verbose:
 167             for suite in self.suitesfailed:
 168                 print "=" * 78
 169                 print "FAIL: %s" % suite
 170                 if suite in self.test_output:
 171                     print self.test_output[suite]
 172                 print ""
 173
 174         f.write("= Skipped tests =\n")
 175         for reason in self.skips.keys():
 176             f.write(reason + "\n")
 177             for name in self.skips[reason]:
 178                 f.write("\t%s\n" % name)
 179             f.write("\n")
 180         f.close()
 181
 182         print "\nA summary with detailed information can be found in:"
 183         print "  %s" % self.summaryfile
 184
 185         if not self.suitesfailed:
 186             ok = (self.statistics['TESTS_EXPECTED_OK'] +
 187                   self.statistics['TESTS_EXPECTED_FAIL'])
 188             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 189         else:
 190             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (
 191                 self.statistics['TESTS_UNEXPECTED_FAIL'],
 192                 self.statistics['TESTS_ERROR'],
 193                 len(self.suitesfailed))
 194
 195     def skip_testsuite(self, name, reason="UNKNOWN"):
 196         self.skips.setdefault(reason, []).append(name)
 197         if self.totalsuites:
 198             self.totalsuites-=1
 199
 200 parser = optparse.OptionParser("format-subunit [options]")
 201 parser.add_option("--verbose", action="store_true",
 202     help="Be verbose")
 203 parser.add_option("--immediate", action="store_true",
 204     help="Show failures immediately, don't wait until test run has finished")
 205 parser.add_option("--prefix", type="string", default=".",
 206     help="Prefix to write summary to")
 207
 208 opts, args = parser.parse_args()
 209
 210 statistics = {
 211     'SUITES_FAIL': 0,
 212     'TESTS_UNEXPECTED_OK': 0,
 213     'TESTS_EXPECTED_OK': 0,
 214     'TESTS_UNEXPECTED_FAIL': 0,
 215     'TESTS_EXPECTED_FAIL': 0,
 216     'TESTS_ERROR': 0,
 217     'TESTS_SKIP': 0,
 218 }
 219
 220 def handle_sigint(sig, stack):
 221         sys.exit(0)
 222 signal.signal(signal.SIGINT, handle_sigint)
 223
 224 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose,
 225     opts.immediate, statistics)
 226
 227 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 228
 229 msg_ops.summary()
 230
 231 sys.exit(expected_ret)