selftest --help
-selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
+selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
=head1 DESCRIPTION
The format for the file is, one entry per line:
-TESTSUITE-NAME/TEST-NAME
+TESTSUITE-NAME.TEST-NAME
-=item I<--skip>
+The reason for a test can also be specified, by adding a hash sign (#) and the reason
+after the test name.
-Specify a file containing a list of tests that should be skipped. Possible candidates are
-tests that segfault the server, flip or don't end.
+=item I<--exclude>
+
+Specify a file containing a list of tests that should be skipped. Possible
+candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
+for the --expected-failures flag.
+
+=item I<--include>
+
+Specify a file containing a list of tests that should be run. Same format
+as the --exclude flag.
+
+Not includes specified means all tests will be run.
=item I<--one>
Abort as soon as one test fails.
+=item I<--testlist>
+
+Load a list of tests from the specified location.
+
=back
=head1 ENVIRONMENT
use POSIX;
use Cwd qw(abs_path);
use lib "$RealBin";
-use Samba3;
-use Samba4;
+use Subunit qw(parse_results);
use SocketWrapper;
my $opt_help = 0;
my $opt_one = 0;
my $opt_immediate = 0;
my $opt_expected_failures = undef;
-my $opt_skip = undef;
+my @opt_exclude = ();
+my @opt_include = ();
my $opt_verbose = 0;
my $opt_testenv = 0;
my $ldap = undef;
my $opt_resetup_env = undef;
my $opt_bindir = undef;
my $opt_no_lazy_setup = undef;
+my $opt_format = "plain";
+my @testlists = ();
my $srcdir = ".";
my $builddir = ".";
my $prefix = "./st";
-my $suitesfailed = [];
-my $start = time();
my @expected_failures = ();
-my @skips = ();
+my @includes = ();
+my @excludes = ();
my $statistics = {
SUITES_FAIL => 0,
- SUITES_OK => 0,
- SUITES_SKIPPED => 0,
TESTS_UNEXPECTED_OK => 0,
TESTS_EXPECTED_OK => 0,
TESTS_UNEXPECTED_FAIL => 0,
TESTS_EXPECTED_FAIL => 0,
- TESTS_ERROR => 0
+ TESTS_ERROR => 0,
+ TESTS_SKIP => 0,
};
-sub expecting_failure($)
-{
- my $fullname = shift;
-
- foreach (@expected_failures) {
- return 1 if ($fullname =~ /$_/);
- }
-
- return 0;
-}
-
-sub skip($)
+sub find_in_list($$)
{
- my $fullname = shift;
-
- foreach (@skips) {
- return 1 if ($fullname =~ /$_/);
- }
-
- return 0;
-}
-
-sub getlog_env($);
-
-my $test_output = {};
-
-sub buildfarm_start_testsuite($)
-{
- my ($state) = @_;
- my $out = "";
-
- $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
- $out .= "Running test $state->{NAME} (level 0 stdout)\n";
- $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
- $out .= scalar(localtime())."\n";
- $out .= "SELFTEST RUNTIME: " . ($state->{START} - $start) . "s\n";
- $out .= "NAME: $state->{NAME}\n";
- $out .= "CMD: $state->{CMD}\n";
-
- $test_output->{$state->{NAME}} = "";
-
- print $out;
-}
-
-sub buildfarm_output_msg($$)
-{
- my ($state, $output) = @_;
-
- $test_output->{$state->{NAME}} .= $output;
-}
-
-sub buildfarm_end_testsuite($$$)
-{
- my ($state, $expected_ret, $ret) = @_;
- my $out = "";
-
- $out .= "TEST RUNTIME: " . (time() - $state->{START}) . "s\n";
-
- if ($ret == $expected_ret) {
- $out .= "ALL OK\n";
- } else {
- $out .= "ERROR: $ret";
- $out .= $test_output->{$state->{NAME}};
- }
+ my ($list, $fullname) = @_;
- $out .= "PCAP FILE: $state->{PCAP_FILE}\n" if defined($state->{PCAP_FILE});
-
- $out .= getlog_env($state->{ENVNAME});
-
- $out .= "==========================================\n";
- if ($ret == $expected_ret) {
- $out .= "TEST PASSED: $state->{NAME}\n";
- } else {
- $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
+ foreach (@$list) {
+ if ($fullname =~ /$$_[0]/) {
+ return ($$_[1]) if ($$_[1]);
+ return "NO REASON SPECIFIED";
+ }
}
- $out .= "==========================================\n";
-
- print $out;
-}
-sub buildfarm_start_test($$)
-{
- my ($state, $testname) = @_;
-}
-
-sub buildfarm_end_test($$$$)
-{
- my ($state, $testname, $result, $expected) = @_;
+ return undef;
}
-my $buildfarm_msg_ops = {
- start_testsuite => \&buildfarm_start_testsuite,
- output_msg => \&buildfarm_output_msg,
- end_testsuite => \&buildfarm_end_testsuite,
- start_test => \&buildfarm_start_test,
- end_test => \&buildfarm_end_test,
-};
-
-sub plain_output_msg($$);
-
-sub plain_start_testsuite($)
-{
- my ($state) = @_;
- my $out = "";
-
- $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
- $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
- $out .= "] $state->{NAME}\n";
-
- $test_output->{$state->{NAME}} = "" unless $opt_verbose;
-
- plain_output_msg($state, "CMD: $state->{CMD}\n");
-
- print $out;
-}
-
-sub plain_output_msg($$)
+sub expecting_failure($)
{
- my ($state, $output) = @_;
-
- if ($opt_verbose) {
- print $output;
- } else {
- $test_output->{$state->{NAME}} .= $output;
- }
+ my ($name) = @_;
+ return find_in_list(\@expected_failures, $name);
}
-sub plain_end_testsuite($$$)
-{
- my ($state, $expected_ret, $ret) = @_;
- my $out = "";
-
- if ($ret != $expected_ret) {
- plain_output_msg($state, "ERROR: $ret\n");
- }
-
- if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
- $out .= $test_output->{$state->{NAME}};
- }
-
- if (not $opt_socket_wrapper_keep_pcap and defined($state->{PCAP_FILE})) {
- $out .= "PCAP FILE: $state->{PCAP_FILE}\n";
- }
-
- $out .= getlog_env($state->{ENVNAME});
-
- print $out;
-}
-
-sub plain_start_test($$)
+sub skip($)
{
- my ($state, $testname) = @_;
-}
+ my ($name) = @_;
-sub plain_end_test($$$$)
-{
- my ($state, $testname, $result, $expected) = @_;
+ return find_in_list(\@excludes, $name);
}
-my $plain_msg_ops = {
- start_testsuite => \&plain_start_testsuite,
- output_msg => \&plain_output_msg,
- end_testsuite => \&plain_end_testsuite,
- start_test => \&plain_start_test,
- end_test => \&plain_end_test,
-};
+sub getlog_env($);
sub setup_pcap($)
{
- my ($state) = @_;
+ my ($state, $name) = @_;
return unless ($opt_socket_wrapper_pcap);
return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
- my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
+ my $fname = $name;
$fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
- $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
+ my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
+
+ SocketWrapper::setup_pcap($pcap_file);
- SocketWrapper::setup_pcap($state->{PCAP_FILE});
+ return $pcap_file;
}
sub cleanup_pcap($$$)
{
- my ($state, $expected_ret, $ret) = @_;
+ my ($pcap_file, $expected_ret, $ret) = @_;
return unless ($opt_socket_wrapper_pcap);
return if ($opt_socket_wrapper_keep_pcap);
return unless ($expected_ret == $ret);
- return unless defined($state->{PCAP_FILE});
+ return unless defined($pcap_file);
- unlink($state->{PCAP_FILE});
- $state->{PCAP_FILE} = undef;
+ unlink($pcap_file);
}
-sub run_test($$$$$$)
+sub run_testsuite($$$$$$)
{
my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
- my $expected_ret = 1;
- my $open_tests = {};
- my $msg_state = {
- ENVNAME => $envname,
- NAME => $name,
- CMD => $cmd,
- INDEX => $i,
- TOTAL => $totalsuites,
- START => time()
- };
+ my $pcap_file = setup_pcap($name);
- setup_pcap($msg_state);
-
- $msg_ops->{start_testsuite}->($msg_state);
+ $msg_ops->start_test([], $name);
open(RESULT, "$cmd 2>&1|");
- while (<RESULT>) {
- $msg_ops->{output_msg}->($msg_state, $_);
- if (/^test: (.+)\n/) {
- $open_tests->{$1} = 1;
- $msg_ops->{start_test}->($msg_state, $1);
- } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
- my $result = $1;
- if ($1 eq "success") {
- delete $open_tests->{$2};
- if (expecting_failure("$name/$2")) {
- $statistics->{TESTS_UNEXPECTED_OK}++;
- $msg_ops->{end_test}->($msg_state, $2, $1, 1);
- } else {
- $statistics->{TESTS_EXPECTED_OK}++;
- $msg_ops->{end_test}->($msg_state, $2, $1, 0);
- }
- } elsif ($1 eq "failure") {
- delete $open_tests->{$2};
- if (expecting_failure("$name/$2")) {
- $statistics->{TESTS_EXPECTED_FAIL}++;
- $msg_ops->{end_test}->($msg_state, $2, $1, 0);
- $expected_ret = 0;
- } else {
- print "n:$name/$2\n";
- $statistics->{TESTS_UNEXPECTED_FAIL}++;
- $msg_ops->{end_test}->($msg_state, $2, $1, 1);
- }
- } elsif ($1 eq "skip") {
- delete $open_tests->{$2};
- $msg_ops->{end_test}->($msg_state, $2, $1, 0);
- } elsif ($1 eq "error") {
- $statistics->{TESTS_ERROR}++;
- delete $open_tests->{$2};
- $msg_ops->{end_test}->($msg_state, $2, $1, 1);
- }
- }
- }
+ my $expected_ret = parse_results(
+ $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
+
+ my $envlog = getlog_env($envname);
+ $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
+
+ $msg_ops->output_msg("CMD: $cmd\n");
- foreach (keys %$open_tests) {
- $msg_ops->{end_test}->($msg_state, $_, "error", 1);
- $msg_ops->{output_msg}->($msg_state, "$_ was started but never finished!");
- $statistics->{TESTS_ERROR}++;
- }
my $ret = close(RESULT);
+ $ret = 0 unless $ret == 1;
+
+ if ($ret == 1) {
+ $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
+ } else {
+ $msg_ops->end_test([], $name, "failure", $expected_ret != $ret,
+ "Returned $ret");
+ }
- cleanup_pcap($msg_state, $expected_ret, $ret);
+ cleanup_pcap($pcap_file, $expected_ret, $ret);
- $msg_ops->{end_testsuite}->($msg_state, $expected_ret, $ret);
+ if (not $opt_socket_wrapper_keep_pcap and
+ defined($pcap_file)) {
+ $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
+ }
if ($ret != $expected_ret) {
- push(@$suitesfailed, $name);
$statistics->{SUITES_FAIL}++;
exit(1) if ($opt_one);
- } else {
- $statistics->{SUITES_OK}++;
}
return ($ret == $expected_ret);
Generic options:
--help this help page
--target=samba4|samba3|win Samba version to target
+ --testlist=FILE file to read available tests from
Paths:
--prefix=DIR prefix to run tests in [st]
--builddir=DIR output directory [.]
Target Specific:
- --socket-wrapper-pcap=DIR save traffic to pcap directories
+ --socket-wrapper-pcap save traffic to pcap directories
--socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
failed
--socket-wrapper enable socket wrapper
'one' => \$opt_one,
'immediate' => \$opt_immediate,
'expected-failures=s' => \$opt_expected_failures,
- 'skip=s' => \$opt_skip,
+ 'exclude=s' => \@opt_exclude,
+ 'include=s' => \@opt_include,
'srcdir=s' => \$srcdir,
'builddir=s' => \$builddir,
'verbose' => \$opt_verbose,
'no-lazy-setup' => \$opt_no_lazy_setup,
'resetup-environment' => \$opt_resetup_env,
'bindir:s' => \$opt_bindir,
+ 'format=s' => \$opt_format,
+ 'testlist=s' => \@testlists
);
exit(1) if (not $result);
# Backwards compatibility:
if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
- if (defined($ENV{FEDORA_DS_PREFIX})) {
+ if (defined($ENV{FEDORA_DS_ROOT})) {
$ldap = "fedora-ds";
} else {
$ldap = "openldap";
die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
$ENV{PREFIX} = $prefix;
+$ENV{KRB5CCNAME} = "$prefix/krb5ticket";
$ENV{PREFIX_ABS} = $prefix_abs;
$ENV{SRCDIR} = $srcdir;
$ENV{SRCDIR_ABS} = $srcdir_abs;
-my $tls_enabled = not $opt_quick;
-my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
- ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
+if (defined($ENV{RUN_FROM_BUILD_FARM}) and
+ ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
+ $opt_format = "buildfarm";
+}
+my $tls_enabled = not $opt_quick;
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
-$ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
+$ENV{LDB_MODULES_PATH} = "$old_pwd/bin/modules/ldb";
$ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
-if (defined($ENV{LD_LIBRARY_PATH})) {
- $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
-} else {
- $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
-}
if (defined($ENV{PKG_CONFIG_PATH})) {
$ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
} else {
$ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
}
-$ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
+# Required for smbscript:
+$ENV{PATH} = "$old_pwd/bin:$old_pwd:$ENV{PATH}";
+if ($opt_socket_wrapper_keep_pcap) {
+ # Socket wrapper keep pcap implies socket wrapper pcap
+ $opt_socket_wrapper_pcap = 1;
+}
if ($opt_socket_wrapper_pcap) {
# Socket wrapper pcap implies socket wrapper
}
my $target;
+my $testenv_default = "none";
if ($opt_target eq "samba4") {
- $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
+ $testenv_default = "member";
+ require target::Samba4;
+ $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
} elsif ($opt_target eq "samba3") {
- if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
+ if ($opt_socket_wrapper and `$opt_bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
}
-
+ $testenv_default = "dc";
+ require target::Samba3;
$target = new Samba3($opt_bindir);
} elsif ($opt_target eq "win") {
die("Windows tests will not run with socket wrapper enabled.")
if ($opt_socket_wrapper);
+ $testenv_default = "dc";
+ require target::Windows;
$target = new Windows();
}
-if (defined($opt_expected_failures)) {
- open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
- while (<KNOWN>) {
+sub read_test_regexes($)
+{
+ my ($name) = @_;
+ my @ret = ();
+ open(LF, "<$name") or die("unable to read $name: $!");
+ while (<LF>) {
chomp;
- s/([ \t]+)\#(.*)$//;
- push (@expected_failures, $_); }
- close(KNOWN);
+ if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
+ push (@ret, [$1, $4]);
+ } else {
+ s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
+ push (@ret, [$_, undef]);
+ }
+ }
+ close(LF);
+ return @ret;
}
-if (defined($opt_skip)) {
- open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
- while (<SKIP>) {
- chomp;
- s/([ \t]+)\#(.*)$//;
- push (@skips, $_); }
- close(SKIP);
+if (defined($opt_expected_failures)) {
+ @expected_failures = read_test_regexes($opt_expected_failures);
+}
+
+foreach (@opt_exclude) {
+ push (@excludes, read_test_regexes($_));
+}
+
+if ($opt_quick) {
+ push (@includes, read_test_regexes("samba4-quick"));
+}
+
+foreach (@opt_include) {
+ push (@includes, read_test_regexes($_));
}
my $interfaces = join(',', ("127.0.0.6/8",
torture:basedir = $prefix_abs/client
#We don't want to pass our self-tests if the PAC code is wrong
gensec:require_pac = true
+ modules dir = $ENV{LD_SAMBA_MODULE_PATH}
";
close(CF);
}
-
my @torture_options = ();
push (@torture_options, "--configfile=$conffile");
# ensure any one smbtorture call doesn't run too long
push (@torture_options, "--maximum-runtime=$torture_maxtime");
push (@torture_options, "--target=$opt_target");
-push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
+push (@torture_options, "--basedir=$prefix_abs");
+push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
push (@torture_options, "--format=subunit");
push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
my %required_envs = ();
-if ($opt_quick) {
- open(IN, "$testsdir/tests_quick.sh|");
-} else {
- open(IN, "$testsdir/tests_all.sh|");
-}
-while (<IN>) {
- if ($_ eq "-- TEST --\n") {
- my $name = <IN>;
- $name =~ s/\n//g;
- my $env = <IN>;
- $env =~ s/\n//g;
- my $cmdline = <IN>;
- $cmdline =~ s/\n//g;
- if (not defined($tests) or $name =~ /$tests/) {
- $required_envs{$env} = 1;
- push (@todo, [$name, $env, $cmdline]);
+sub read_testlist($)
+{
+ my ($filename) = @_;
+
+ my @ret = ();
+ open(IN, $filename) or die("Unable to open $filename: $!");
+
+ while (<IN>) {
+ if ($_ eq "-- TEST --\n") {
+ my $name = <IN>;
+ $name =~ s/\n//g;
+ my $env = <IN>;
+ $env =~ s/\n//g;
+ my $cmdline = <IN>;
+ $cmdline =~ s/\n//g;
+ if (not defined($tests) or $name =~ /$tests/) {
+ $required_envs{$env} = 1;
+ push (@ret, [$name, $env, $cmdline]);
+ }
+ } else {
+ print;
}
+ }
+ close(IN) or die("Error creating recipe");
+ return @ret;
+}
+
+if ($#testlists == -1) {
+ die("No testlists specified");
+}
+
+my @available = ();
+foreach my $fn (@testlists) {
+ foreach (read_testlist($fn)) {
+ my $name = $$_[0];
+ next if (@includes and not find_in_list(\@includes, $name));
+ push (@available, $_);
+ }
+}
+
+my $msg_ops;
+if ($opt_format eq "buildfarm") {
+ require output::buildfarm;
+ $msg_ops = new output::buildfarm($statistics);
+} elsif ($opt_format eq "plain") {
+ require output::plain;
+ $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
+} elsif ($opt_format eq "html") {
+ require output::html;
+ mkdir("test-results", 0777);
+ $msg_ops = new output::html("test-results", $statistics);
+} else {
+ die("Invalid output format '$opt_format'");
+}
+
+
+foreach (@available) {
+ my $name = $$_[0];
+ my $skipreason = skip($name);
+ if ($skipreason) {
+ $msg_ops->skip_testsuite($name, $skipreason);
} else {
- print;
+ push(@todo, $_);
}
}
-close(IN) or die("Error creating recipe");
+
+if ($#todo == -1) {
+ print STDERR "No tests to run\n";
+ exit(1);
+ }
my $suitestotal = $#todo + 1;
my $i = 0;
# misc stuff
"KRB5_CONFIG",
- "WINBINDD_SOCKET_DIR"
+ "WINBINDD_SOCKET_DIR",
+ "WINBINDD_PRIV_PIPE_DIR"
);
+$SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
+ my $signame = shift;
+ teardown_env($_) foreach(keys %running_envs);
+ die("Received signal $signame");
+};
+
sub setup_env($)
{
my ($envname) = @_;
return undef unless defined($testenv_vars);
+ $running_envs{$envname} = $testenv_vars;
+
SocketWrapper::set_default_iface(6);
write_clientconf($conffile, $testenv_vars);
}
}
- $running_envs{$envname} = $testenv_vars;
return $testenv_vars;
}
delete $running_envs{$envname};
}
-my $msg_ops;
-if ($from_build_farm) {
- $msg_ops = $buildfarm_msg_ops;
-} else {
- $msg_ops = $plain_msg_ops;
-}
-
if ($opt_no_lazy_setup) {
setup_env($_) foreach (keys %required_envs);
}
if ($opt_testenv) {
my $testenv_name = $ENV{SELFTEST_TESTENV};
- $testenv_name = "dc" unless defined($testenv_name);
+ $testenv_name = $testenv_default unless defined($testenv_name);
my $testenv_vars = setup_env($testenv_name);
CONFIGURATION=\$CONFIGURATION
$envvarstr
-\" && bash'");
+\" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
teardown_env($testenv_name);
} else {
foreach (@todo) {
my $name = $$_[0];
my $envname = $$_[1];
- if (skip($name)) {
- print "SKIPPED: $name\n";
- $statistics->{SUITES_SKIPPED}++;
- next;
- }
-
my $envvars = setup_env($envname);
if (not defined($envvars)) {
- push(@$suitesfailed, $name);
- $statistics->{SUITES_FAIL}++;
- $statistics->{TESTS_ERROR}++;
- print "FAIL: $name (ENV[$envname] not available!)\n";
+ $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
next;
}
- run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
+ run_testsuite($envname, $name, $cmd, $i, $suitestotal,
+ $msg_ops);
if (defined($opt_analyse_cmd)) {
system("$opt_analyse_cmd \"$name\"");
$target->stop();
-my $end = time();
-my $duration = ($end-$start);
-my $numfailed = $#$suitesfailed+1;
-if ($numfailed == 0) {
- my $ok = $statistics->{TESTS_EXPECTED_OK} +
- $statistics->{TESTS_EXPECTED_FAIL};
- print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
-} else {
- unless ($from_build_farm) {
- if (not $opt_immediate and not $opt_verbose) {
- foreach (@$suitesfailed) {
- print "===============================================================================\n";
- print "FAIL: $_\n";
- print $test_output->{$_};
- print "\n";
- }
- }
-
- print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
- }
-}
-print "DURATION: $duration seconds\n";
+$msg_ops->summary();
my $failed = 0;
}
}
-if ($from_build_farm) {
- print "TEST STATUS: $numfailed\n";
+if ($opt_format eq "buildfarm") {
+ print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
}
-exit $numfailed;
+exit $statistics->{SUITES_FAIL};