selftest/format-subunit

   1 #!/usr/bin/env python
   2 # vim: expandtab
   3 # Pretty-format subunit output
   4 # Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
   5 # Published under the GNU GPL, v3 or later
   6
   7 import optparse
   8 import os
   9 import sys
  10
  11 import subunithelper
  12
  13 class PlainFormatter(object):
  14
  15     def __init__(self, summaryfile, verbose, immediate, statistics, totaltests=None):
  16         self.verbose = verbose
  17         self.immediate = immediate
  18         self.statistics = statistics
  19         self.start_time = None
  20         self.test_output = {}
  21         self.suitesfailed = []
  22         self.suites_ok = 0
  23         self.skips = {}
  24         self.summaryfile = summaryfile
  25         self.index = 0
  26         self.NAME = None
  27         self.totalsuites = totaltests
  28
  29     def testsuite_count(self, count):
  30         self.totalsuites = count
  31
  32     def report_time(self, time):
  33         if self.start_time is None:
  34             self.start_time = time
  35         self.last_time = time
  36
  37     def start_testsuite(self, name):
  38         self.index += 1
  39         self.NAME = name
  40         self.START_TIME = self.last_time
  41
  42         duration = self.START_TIME - self.start_time
  43
  44         if not self.verbose:
  45             self.test_output[name] = ""
  46
  47         out = ""
  48         out += "[%d" % self.index
  49         if self.totalsuites is not None:
  50             out += "/%d" % self.totalsuites
  51         out += " in %ds" % duration
  52         if self.suitesfailed:
  53             out += ", %d errors" % (len(self.suitesfailed),)
  54         out += "] %s" % name
  55         if self.immediate:
  56             sys.stdout.write(out + "\n")
  57         else:
  58             sys.stdout.write(out + ": ")
  59
  60     def output_msg(self, output):
  61         if self.verbose:
  62             sys.stdout.write(output)
  63         elif self.NAME is not None:
  64             self.test_output[self.NAME] += output
  65         else:
  66             sys.stdout.write(output)
  67
  68     def control_msg(self, output):
  69         #$self->output_msg($output)
  70         pass
  71
  72     def end_testsuite(self, name, result, reason):
  73         out = ""
  74         unexpected = 0
  75
  76         if not name in self.test_output:
  77             print "no output for name[%s]" % name
  78
  79         if result in ("success", "xfail"):
  80             self.suites_ok+=1
  81         else:
  82             self.output_msg("ERROR: Testsuite[%s]\nREASON: %s\n" % (name, reason))
  83             self.suitesfailed.append(name)
  84             if self.immediate and not self.verbose:
  85                 out += self.test_output[name]
  86             unexpected = 1
  87
  88         if not self.immediate:
  89             if not unexpected:
  90                 out += " ok\n"
  91             else:
  92                 out += " " + result.upper() + "\n"
  93
  94         sys.stdout.write(out)
  95
  96     def start_test(self, testname):
  97         pass
  98
  99     def end_test(self, testname, result, unexpected, reason):
 100         append = ""
 101
 102         if not unexpected:
 103             self.test_output[self.NAME] = ""
 104             if not self.immediate:
 105                 sys.stdout.write({
 106                     'failure': 'f',
 107                     'xfail': 'X',
 108                     'skip': 's',
 109                     'success': '.'}.get(result, "?(%s)" % result))
 110             return
 111
 112         reason = reason.strip()
 113
 114         append = "UNEXPECTED(%s): %s\nREASON: %s\n" % (result, testname, reason)
 115
 116         self.test_output[self.NAME] += append
 117
 118         if self.immediate and not self.verbose:
 119             print self.test_output[self.NAME]
 120             self.test_output[self.NAME] = ""
 121
 122         if not self.immediate:
 123             sys.stdout.write({
 124                'error': 'E',
 125                'failure': 'F',
 126                'success': 'S'}.get(result, "?"))
 127
 128     def summary(self):
 129         f = open(self.summaryfile, 'w+')
 130
 131         if self.suitesfailed:
 132             f.write("= Failed tests =\n")
 133
 134             for suite in self.suitesfailed:
 135                 f.write("== %s ==\n" % suite)
 136                 f.write(self.test_output[suite]+"\n\n")
 137
 138             f.write("\n")
 139
 140         if not self.immediate and not self.verbose:
 141             for suite in self.suitesfailed:
 142                 print "==============================================================================="
 143                 print "FAIL: %s" % suite
 144                 print self.test_output[suite]
 145                 print ""
 146
 147         f.write("= Skipped tests =\n")
 148         for reason in self.skips.keys():
 149             f.write(reason + "\n")
 150             for name in self.skips[reason]:
 151                 f.write("\t%s\n" % name)
 152             f.write("\n")
 153         f.close()
 154
 155         print "\nA summary with detailed information can be found in:\n  %s\n" % self.summaryfile
 156
 157         if not self.suitesfailed:
 158             ok = self.statistics['TESTS_EXPECTED_OK'] + self.statistics['TESTS_EXPECTED_FAIL']
 159             print "\nALL OK (%d tests in %d testsuites)" % (ok, self.suites_ok)
 160         else:
 161             print "\nFAILED (%d failures and %d errors in %d testsuites)" % (self.statistics['TESTS_UNEXPECTED_FAIL'], self.statistics['TESTS_ERROR'], len(self.suitesfailed))
 162
 163     def skip_testsuite(self, name, reason="UNKNOWN"):
 164         self.skips.setdefault(reason, []).append(name)
 165         if self.totalsuites:
 166             self.totalsuites-=1
 167
 168 parser = optparse.OptionParser("format-subunit [options]")
 169 parser.add_option("--verbose", action="store_true",
 170     help="Be verbose")
 171 parser.add_option("--immediate", action="store_true",
 172     help="Show failures immediately, don't wait until test run has finished")
 173 parser.add_option("--prefix", type="string", default=".",
 174     help="Prefix to write summary to")
 175
 176 opts, args = parser.parse_args()
 177
 178 statistics = {
 179     'SUITES_FAIL': 0,
 180     'TESTS_UNEXPECTED_OK': 0,
 181     'TESTS_EXPECTED_OK': 0,
 182     'TESTS_UNEXPECTED_FAIL': 0,
 183     'TESTS_EXPECTED_FAIL': 0,
 184     'TESTS_ERROR': 0,
 185     'TESTS_SKIP': 0,
 186 }
 187
 188 msg_ops = PlainFormatter(os.path.join(opts.prefix, "summary"), opts.verbose, opts.immediate, statistics)
 189
 190 expected_ret = subunithelper.parse_results(msg_ops, statistics, sys.stdin)
 191
 192 msg_ops.summary()
 193
 194 sys.exit(expected_ret)