selftest: Fix handling of testsuite, reintroduce progress indication.

diff --git a/lib/subunit/tap2subunit b/lib/subunit/tap2subunit
index 45f515540b0975738c060b24839f0fd873d1ae57..9e335168f522f4aeefda876cd4d58c341ffd0a3e 100755 (executable)
--- a/lib/subunit/tap2subunit
+++ b/lib/subunit/tap2subunit
@@ -33,4 +33,3 @@ while(<STDIN>) {
        }
 }
 exit $error;
-

diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm
index 5ee8743c6c5828eb9768ccbf77f5aaf8dd157345..4fddeec8b131c9d6fee152dedb89ac6f448d14ee 100644 (file)
--- a/selftest/Subunit.pm
+++ b/selftest/Subunit.pm
@@ -35,11 +35,11 @@ sub parse_results($$$$)
        while(<$fh>) {
                if (/^test: (.+)\n/) {
                        $msg_ops->control_msg($_);
-                       $msg_ops->start_test($open_tests, $1);
+                       $msg_ops->start_test($1);
                        push (@$open_tests, $1);
                } elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) {
                        $msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1));
-               } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) {
+               } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) {
                        $msg_ops->control_msg($_);
                        my $result = $1;
                        my $testname = $2;
@@ -55,7 +55,7 @@ sub parse_results($$$$)
                                
                                unless ($terminated) {
                                        $statistics->{TESTS_ERROR}++;
-                                       $msg_ops->end_test($testname, $result, 1, "reason interrupted");
+                                       $msg_ops->end_test($testname, "error", 1, "reason ($result) interrupted");
                                        return 1;
                                }
                        }
@@ -85,14 +85,26 @@ sub parse_results($$$$)
                                pop(@$open_tests); #FIXME: Check that popped value == $testname
                                $msg_ops->end_test($testname, $result, 1, $reason);
                                $unexpected_err++;
-                       }
+                       } elsif ($result eq "skip-testsuite") {
+                               $msg_ops->skip_testsuite($testname);
+                       } elsif ($result eq "testsuite-success") {
+                               $msg_ops->end_testsuite($testname, "success", $reason);
+                       } elsif ($result eq "testsuite-failure") {
+                               $msg_ops->end_testsuite($testname, "failure", $reason);
+                       } elsif ($result eq "testsuite-error") {
+                               $msg_ops->end_testsuite($testname, "error", $reason);
+                       } 
+               } elsif (/^testsuite: (.*)\n/) {
+                       $msg_ops->start_testsuite($1);
+               } elsif (/^testsuite-count: (\d+)\n/) {
+                       $msg_ops->testsuite_count($1);
                } else {
                        $msg_ops->output_msg($_);
                }
        }
 
        while ($#$open_tests > $orig_open_len) {
-               $msg_ops->end_test($open_tests, pop(@$open_tests), "error", 1,
+               $msg_ops->end_test(pop(@$open_tests), "error", 1,
                                   "was started but never finished!");
                $statistics->{TESTS_ERROR}++;
                $unexpected_err++;
@@ -118,7 +130,7 @@ sub end_test($$;$)
        my $result = shift;
        my $reason = shift;
        if ($reason) {
-               print "$result: $name [";
+               print "$result: $name [\n";
                print "$reason";
                print "]\n";
        } else {
@@ -126,6 +138,34 @@ sub end_test($$;$)
        }
 }
 
+sub skip_test($;$)
+{
+       my $name = shift;
+       my $reason = shift;
+       end_test($name, "skip", $reason);
+}
+
+sub fail_test($;$)
+{
+       my $name = shift;
+       my $reason = shift;
+       end_test($name, "fail", $reason);
+}
+
+sub success_test($;$)
+{
+       my $name = shift;
+       my $reason = shift;
+       end_test($name, "success", $reason);
+}
+
+sub xfail_test($;$)
+{
+       my $name = shift;
+       my $reason = shift;
+       end_test($name, "xfail", $reason);
+}
+
 sub report_time($)
 {
        my ($time) = @_;
@@ -133,4 +173,42 @@ sub report_time($)
        printf "time: %04d-%02d-%02d %02d:%02d:%02dZ\n", $year+1900, $mon, $mday, $hour, $min, $sec;
 }
 
+# The following are Samba extensions:
+
+sub start_testsuite($)
+{
+       my ($name) = @_;
+       print "testsuite: $name\n";
+}
+
+sub skip_testsuite($;$)
+{
+       my ($name, $reason) = @_;
+       if ($reason) {
+               print "skip-testsuite: $name [$reason]\n";
+       } else {
+               print "skip-testsuite: $name\n";
+       }
+}
+
+sub end_testsuite($$;$)
+{
+       my $name = shift;
+       my $result = shift;
+       my $reason = shift;
+       if ($reason) {
+               print "testsuite-$result: $name [";
+               print "$reason";
+               print "]\n";
+       } else {
+               print "$result: $name\n";
+       }
+}
+
+sub testsuite_count($)
+{
+       my ($count) = @_;
+       print "testsuite-count: $count\n";
+}
+
 1;

diff --git a/selftest/filter-subunit.pl b/selftest/filter-subunit.pl
index 9a2c6f556c70b4d43e752cce952f033c2775696e..b7a72217f3beccfb85a6f2575c5e3f1ebaa52458 100755 (executable)
--- a/selftest/filter-subunit.pl
+++ b/selftest/filter-subunit.pl
@@ -147,9 +147,9 @@ sub output_msg($$)
        print $msg;
 }
 
-sub start_test($$$)
+sub start_test($$)
 {
-       my ($self, $parents, $testname) = @_;
+       my ($self, $testname) = @_;
 
        if (defined($opt_prefix)) {
                $testname = $opt_prefix.$testname;
@@ -160,7 +160,7 @@ sub start_test($$$)
 
 sub end_test($$$$$)
 {
-       my ($self, $parents, $testname, $result, $unexpected, $reason) = @_;
+       my ($self, $testname, $result, $unexpected, $reason) = @_;
 
        if (defined($opt_prefix)) {
                $testname = $opt_prefix.$testname;
@@ -174,6 +174,29 @@ sub end_test($$$$$)
        Subunit::end_test($testname, $result, $reason);
 }
 
+sub skip_testsuite($;$)
+{
+       Subunit::skip_testsuite(@_);
+}
+
+sub start_testsuite($;$)
+{
+       my ($self, $name) = @_;
+       Subunit::start_testsuite($name);
+}
+
+sub end_testsuite($$;$)
+{
+       my ($self, $name, $result, $reason) = @_;
+       Subunit::end_testsuite($name, $result, $reason);
+}
+
+sub testsuite_count($$)
+{
+       my ($self, $count) = @_;
+       Subunit::testsuite_count($count);
+}
+
 my $msg_ops = {};
 bless $msg_ops;
 

diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm
index 2427136073f76241ce545f5b507e2f61d6bdbb26..bfdced293645c17bb47e68091fb1091ac1de9984 100644 (file)
--- a/selftest/output/plain.pm
+++ b/selftest/output/plain.pm
@@ -1,6 +1,6 @@
 #!/usr/bin/perl
 # Plain text output for selftest
-# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,6 +41,12 @@ sub new($$$$$$$) {
        bless($self, $class);
 }
 
+sub testsuite_count($$)
+{
+       my ($self, $count) = @_;
+       $self->{totalsuites} = $count;
+}
+
 sub report_time($$)
 {
        my ($self, $time) = @_;
@@ -101,23 +107,21 @@ sub control_msg($$)
        #$self->output_msg($output);
 }
 
-sub end_testsuite($$$$$)
+sub end_testsuite($$$$)
 {
-       my ($self, $name, $result, $unexpected, $reason) = @_;
+       my ($self, $name, $result, $reason) = @_;
        my $out = "";
+       my $unexpected = 0;
 
-       if ($unexpected) {
-               if ($result eq "success" and not defined($reason)) {
-                       $reason = "Expected negative exit code, got positive exit code";
-               } 
+       if ($result eq "success" or $result eq "xfail") {
+               $self->{suites_ok}++;
+       } else {
                $self->output_msg("ERROR: $reason\n");
                push (@{$self->{suitesfailed}}, $name);
-       } else {
-               $self->{suites_ok}++;
-       }
-
-       if ($unexpected and $self->{immediate} and not $self->{verbose}) {
-               $out .= $self->{test_output}->{$name};
+               if ($self->{immediate} and not $self->{verbose}) {
+                       $out .= $self->{test_output}->{$name};
+               }
+               $unexpected = 1;
        }
 
        if (not $self->{immediate}) {

diff --git a/selftest/selftest.pl b/selftest/selftest.pl
index d98c5ae370b9ae1a7b1c1d5195d863f0622ab05b..da2594395745f2ed79f10622fd3e919c7733a8b8 100755 (executable)
--- a/selftest/selftest.pl
+++ b/selftest/selftest.pl
@@ -166,7 +166,7 @@ sub find_in_list($$)
        foreach (@$list) {
                if ($fullname =~ /$$_[0]/) {
                         return ($$_[1]) if ($$_[1]);
-                        return "NO REASON SPECIFIED";
+                        return "";
                }
        }
 
@@ -217,11 +217,15 @@ sub run_testsuite($$$$$)
        my $pcap_file = setup_pcap($name);
 
        Subunit::report_time(time());
-       Subunit::start_test($name);
+       Subunit::start_testsuite($name);
 
-       my $ret = system("$cmd | $RealBin/filter-subunit.pl --prefix \"$name.\" 2>&1");
+       system("$cmd 2>&1 | $RealBin/filter-subunit.pl --prefix \"$name.\"");
+       my $ret = $?;
        if ($ret == -1) {
-               Subunit::end_test($name, "error", "Unable to run $cmd: $!");
+               Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
+               return 0;
+       } elsif ($ret & 127) {
+               Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
                return 0;
        }
        my $envlog = getlog_env($envname);
@@ -241,7 +245,7 @@ sub run_testsuite($$$$$)
        } else {
                $result = "failure";
        }
-       Subunit::end_test($name, $result, $reason);
+       Subunit::end_testsuite($name, $result, $reason);
 
        cleanup_pcap($pcap_file, $exitcode);
 
@@ -616,18 +620,19 @@ my @available = ();
 foreach my $fn (@testlists) {
        foreach (read_testlist($fn)) {
                my $name = $$_[0];
-               next if (@includes and not find_in_list(\@includes, $name));
+               next if (@includes and not defined(find_in_list(\@includes, $name)));
                push (@available, $_);
        }
 }
 
+Subunit::testsuite_count($#available+1);
 Subunit::report_time(time());
 
 foreach (@available) {
        my $name = $$_[0];
        my $skipreason = skip($name);
-       if ($skipreason) {
-               Subunit::end_test($name, "skip", $skipreason);
+       if (defined($skipreason)) {
+               Subunit::skip_testsuite($name, $skipreason);
        } else {
                push(@todo, $_); 
        }
@@ -815,7 +820,7 @@ $envvarstr
                
                my $envvars = setup_env($envname);
                if (not defined($envvars)) {
-                       Subunit::end_test($name, "skip", 
+                       Subunit::skip_testsuite($name, 
                                "unable to set up environment $envname");
                        next;
                }

diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk
index 3271e7031122fab37738b263f35e7e6c7edc6969..328ae093308d7c0d059f467cf21d574926515a2c 100644 (file)
--- a/source4/selftest/config.mk
+++ b/source4/selftest/config.mk
@@ -11,7 +11,7 @@ SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selfte
 FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(srcdir)/selftest/knownfail
 FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT)
 
-subunittest:: everything
+test-subunit:: everything
        $(SELFTEST) --socket-wrapper $(TESTS)
 
 slowtest:: everything
@@ -46,6 +46,9 @@ test-noswrap:: everything
 quicktest:: all
        $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate 
 
+quicktest-subunit:: all
+       $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS)
+
 quicktestone:: all
        $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT)