# Bootstrap Samba and run a number of tests against it.
# Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
# Published under the GNU GPL, v3 or later.
+
+=pod
+
+=head1 NAME
+
+selftest - Samba test runner
+
+=head1 SYNOPSIS
+
+selftest --help
+
+selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
+
+=head1 DESCRIPTION
+
+A simple test runner. TESTS is a regular expression with tests to run.
+
+=head1 OPTIONS
+
+=over 4
+
+=item I<--help>
+
+Show list of available options.
+
+=item I<--srcdir=DIR>
+
+Source directory.
+
+=item I<--builddir=DIR>
+
+Build directory.
+
+=item I<--prefix=DIR>
+
+Change directory to run tests in. Default is 'st'.
+
+=item I<--immediate>
+
+Show errors as soon as they happen rather than at the end of the test run.
+
+=item I<--target samba4|samba3|win>
+
+Specify test target against which to run. Default is 'samba4'.
+
+=item I<--quick>
+
+Run only a limited number of tests. Intended to run in about 30 seconds on
+moderately recent systems.
+
+=item I<--socket-wrapper>
+
+Use socket wrapper library for communication with server. Only works
+when the server is running locally.
+
+Will prevent TCP and UDP ports being opened on the local host but
+(transparently) redirects these calls to use unix domain sockets.
+
+=item I<--expected-failures>
+
+Specify a file containing a list of tests that are expected to fail. Failures for
+these tests will be counted as successes, successes will be counted as failures.
+
+The format for the file is, one entry per line:
+
+TESTSUITE-NAME/TEST-NAME
+
+=item I<--one>
+
+Abort as soon as one test fails.
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 4
+
+=item I<SMBD_VALGRIND>
+
+=item I<TORTURE_MAXTIME>
+
+=item I<VALGRIND>
+
+=item I<TEST_LDAP>
+
+=item I<TLS_ENABLED>
+
+=item I<srcdir>
+
+=back
+
+=head1 LICENSE
+
+selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
+
+=head1 AUTHOR
+
+Jelmer Vernooij
+
+=cut
+
use strict;
use warnings;
use Getopt::Long;
use POSIX;
use Cwd;
+use lib "$RealBin";
+use Samba4;
+use SocketWrapper;
-sub slapd_start($$) {
- my ($conf, $uri) = @_;
- my $oldpath = $ENV{PATH};
- $ENV{PATH} = "/usr/local/sbin:/usr/sbin:/sbin:$ENV{PATH}";
- # running slapd in the background means it stays in the same process group, so it can be
- # killed by timelimit
- system("slapd -d0 -f $conf -h $uri &");
- $ENV{PATH} = $oldpath;
- return $? >> 8;
-}
+my $opt_help = 0;
+my $opt_target = "samba4";
+my $opt_quick = 0;
+my $opt_socket_wrapper = 0;
+my $opt_socket_wrapper_pcap = undef;
+my $opt_one = 0;
+my $opt_immediate = 0;
+my $opt_expected_failures = undef;
+my $opt_verbose = 0;
-sub smbd_check_or_start($$$$$$)
+my $srcdir = ".";
+my $builddir = ".";
+my $prefix = "st";
+
+my $suitesfailed = [];
+my $start = time();
+my @expected_failures = ();
+
+my $statistics = {
+ SUITES_FAIL => 0,
+ SUITES_OK => 0,
+
+ TESTS_UNEXPECTED_OK => 0,
+ TESTS_EXPECTED_OK => 0,
+ TESTS_UNEXPECTED_FAIL => 0,
+ TESTS_EXPECTED_FAIL => 0,
+ TESTS_ERROR => 0
+};
+
+sub expecting_failure($)
{
- my ($bindir, $test_fifo, $test_log, $socket_wrapper_dir, $max_time, $conffile) = @_;
- return 0 if ( -p $test_fifo );
+ my $fullname = shift;
- if (defined($socket_wrapper_dir)) {
- if ( -d $socket_wrapper_dir ) {
- unlink <$socket_wrapper_dir/*>;
- } else {
- mkdir($socket_wrapper_dir);
- }
+ foreach (@expected_failures) {
+ return 1 if $fullname =~ /^$_$/;
}
- unlink($test_fifo);
- system("mkfifo $test_fifo");
-
- unlink($test_log);
-
- my $valgrind = "";
- if (defined($ENV{SMBD_VALGRIND})) {
- $valgrind = $ENV{SMBD_VALGRIND};
- }
-
- print "STARTING SMBD...";
- my $pid = fork();
- if ($pid == 0) {
- my $ret = system("$valgrind $bindir/smbd --maximum-runtime=$max_time -s $conffile -M single -i --leak-report-full < $test_fifo > $test_log");
- open LOG, ">>$test_log";
- if ($? == -1) {
- print LOG "Unable to start smbd: $ret: $!\n";
- print "Unable to start smbd: $ret: $!\n";
- exit 1;
- }
- unlink($test_fifo);
- unlink(<$socket_wrapper_dir/*>) if (defined($socket_wrapper_dir) and -d $socket_wrapper_dir);
- my $exit = $? >> 8;
- if ( $ret == 0 ) {
- print "smbd exits with status $exit\n";
- print LOG "smbd exits with status $exit\n";
- } elsif ( $ret & 127 ) {
- print "smbd got signal ".($ret & 127)." and exits with $exit!\n";
- print LOG "smbd got signal".($ret & 127). " and exits with $exit!\n";
- } else {
- $ret = $? >> 8;
- print "smbd failed with status $exit!\n";
- print LOG "smbd failed with status $exit!\n";
+ return 0;
+}
+
+sub run_test_buildfarm($$$$)
+{
+ my ($name, $cmd, $i, $suitestotal) = @_;
+ print "--==--==--==--==--==--==--==--==--==--==--\n";
+ print "Running test $name (level 0 stdout)\n";
+ print "--==--==--==--==--==--==--==--==--==--==--\n";
+ system("date");
+
+ my $expected_ret = 1;
+ my $open_tests = {};
+ open(RESULT, "$cmd|");
+ while (<RESULT>) {
+ print;
+ if (/^test: (.+)\n/) {
+ $open_tests->{$1} = 1;
+ } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+ my $result = $1;
+ if ($1 eq "success") {
+ delete $open_tests->{$2};
+ if (expecting_failure("$name/$2")) {
+ $statistics->{TESTS_UNEXPECTED_OK}++;
+ } else {
+ $statistics->{TESTS_EXPECTED_OK}++;
+ }
+ } elsif ($1 eq "failure") {
+ delete $open_tests->{$2};
+ if (expecting_failure("$name/$2")) {
+ $statistics->{TESTS_EXPECTED_FAIL}++;
+ $expected_ret = 0;
+ } else {
+ $statistics->{TESTS_UNEXPECTED_FAIL}++;
+ }
+ } elsif ($1 eq "skip") {
+ delete $open_tests->{$2};
+ } elsif ($1 eq "error") {
+ $statistics->{TESTS_ERROR}++;
+ delete $open_tests->{$2};
+ }
}
- close(LOG);
- exit $exit;
}
- print "DONE\n";
+ print "COMMAND: $cmd\n";
+ foreach (keys %$open_tests) {
+ print "$_ was started but never finished!\n";
+ $statistics->{TESTS_ERROR}++;
+ }
+ my $ret = close(RESULT);
- return $pid;
+ print "==========================================\n";
+ if ($ret == $expected_ret) {
+ print "TEST PASSED: $name\n";
+ } else {
+ print "TEST FAILED: $name (status $ret)\n";
+ }
+ print "==========================================\n";
}
-sub teststatus($$) {
- my ($name, $failed) = @_;
-
- print "TEST STATUS: $failed failures\n";
- if ($failed > 0) {
-print <<EOF
-************************
-*** TESTSUITE FAILED ***
-************************
-EOF
-;
+my $test_output = {};
+sub run_test_plain($$$$)
+{
+ my ($name, $cmd, $i, $totalsuites) = @_;
+ my $err = "";
+ if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
+ printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
+ open(RESULT, "$cmd 2>&1|");
+ my $expected_ret = 1;
+ my $open_tests = {};
+ $test_output->{$name} = "";
+ while (<RESULT>) {
+ $test_output->{$name}.=$_;
+ print if ($opt_verbose);
+ if (/^test: (.+)\n/) {
+ $open_tests->{$1} = 1;
+ } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
+ my $result = $1;
+ if ($1 eq "success") {
+ delete $open_tests->{$2};
+ if (expecting_failure("$name/$2")) {
+ $statistics->{TESTS_UNEXPECTED_OK}++;
+ } else {
+ $statistics->{TESTS_EXPECTED_OK}++;
+ }
+ } elsif ($1 eq "failure") {
+ delete $open_tests->{$2};
+ if (expecting_failure("$name/$2")) {
+ $statistics->{TESTS_EXPECTED_FAIL}++;
+ $expected_ret = 0;
+ } else {
+ $statistics->{TESTS_UNEXPECTED_FAIL}++;
+ }
+ } elsif ($1 eq "skip") {
+ delete $open_tests->{$2};
+ } elsif ($1 eq "error") {
+ $statistics->{TESTS_ERROR}++;
+ delete $open_tests->{$2};
+ }
+ }
+ }
+ $test_output->{$name}.="COMMAND: $cmd\n";
+ foreach (keys %$open_tests) {
+ $test_output->{$name}.="$_ was started but never finished!\n";
+ $statistics->{TESTS_ERROR}++;
+ }
+ my $ret = close(RESULT);
+ if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
+ print "$test_output->{$name}\n";
+ }
+ if ($ret != $expected_ret) {
+ push(@$suitesfailed, $name);
+ $statistics->{SUITES_FAIL}++;
+ exit(1) if ($opt_one);
+ } else {
+ $statistics->{SUITES_OK}++;
}
- exit $failed;
}
sub ShowHelp()
print "Samba test runner
Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
-Usage: $Script PREFIX
+Usage: $Script [OPTIONS] PREFIX
Generic options:
--help this help page
+
+Paths:
+ --prefix=DIR prefix to run tests in [st]
+ --srcdir=DIR source directory [.]
+ --builddir=DIR output directory [.]
+
+Target Specific:
--target=samba4|samba3|win Samba version to target
+ --socket-wrapper-pcap=FILE save traffic to pcap file
--socket-wrapper enable socket wrapper
+ --expected-failures=FILE specify list of tests that is guaranteed to fail
+
+Behaviour:
--quick run quick overall test
--one abort when the first test fails
+ --immediate print test output for failed tests during run
+ --verbose be verbose
";
exit(0);
}
-my $opt_help = 0;
-my $opt_target = "samba4";
-my $opt_quick = 0;
-my $opt_socket_wrapper = 0;
-my $opt_one = 0;
-
my $result = GetOptions (
'help|h|?' => \$opt_help,
- 'target' => \$opt_target,
+ 'target=s' => \$opt_target,
+ 'prefix=s' => \$prefix,
'socket-wrapper' => \$opt_socket_wrapper,
+ 'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
'quick' => \$opt_quick,
- 'one' => \$opt_one
+ 'one' => \$opt_one,
+ 'immediate' => \$opt_immediate,
+ 'expected-failures=s' => \$opt_expected_failures,
+ 'srcdir=s' => \$srcdir,
+ 'builddir=s' => \$builddir,
+ 'verbose' => \$opt_verbose
);
-if (not $result) {
- exit(1);
-}
+exit(1) if (not $result);
ShowHelp() if ($opt_help);
-ShowHelp() if ($#ARGV < 0);
-my $prefix = shift;
+my $tests = shift;
my $torture_maxtime = $ENV{TORTURE_MAXTIME};
unless (defined($torture_maxtime)) {
$torture_maxtime = 1200;
}
-# disable rpc validation when using valgrind - its way too slow
-my $valgrind = $ENV{VALGRIND};
-my $validate = undef;
-unless (defined($valgrind)) {
- $validate = "validate";
+# quick hack to disable rpc validation when using valgrind - its way too slow
+unless (defined($ENV{VALGRIND})) {
+ $ENV{VALIDATE} = "validate";
}
my $old_pwd = "$RealBin/../..";
$prefix =~ s+//+/+;
$ENV{PREFIX} = $prefix;
-my $srcdir = "$RealBin/../..";
-if (defined($ENV{srcdir})) {
- $srcdir = $ENV{srcdir};
-}
$ENV{SRCDIR} = $srcdir;
my $bindir = "$srcdir/bin";
my $testsdir = "$srcdir/script/tests";
my $tls_enabled = not $opt_quick;
+my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
+ ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
$ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
$ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
$ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
+my @torture_options = ();
+
+my $testenv_vars = {};
+
if ($opt_target eq "samba4") {
- print "PROVISIONING...";
- open(IN, "$RealBin/mktestsetup.sh $prefix|") or die("Unable to setup");
- while (<IN>) {
- next unless (/^([A-Z_]+)=(.*)$/);
- $ENV{$1} = $2;
- }
- close(IN);
+ $testenv_vars = Samba4::provision($prefix);
} elsif ($opt_target eq "win") {
die ("Windows tests will not run without root privileges.")
if (`whoami` ne "root");
die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
$ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
+} elsif ($opt_target eq "none") {
} else {
die("unknown target `$opt_target'");
}
-my $socket_wrapper_dir = undef;
+foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
+
+if ($opt_socket_wrapper_pcap) {
+ $ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
+ # Socket wrapper pcap implies socket wrapper
+ $opt_socket_wrapper = 1;
+}
-if ( $opt_socket_wrapper)
+my $socket_wrapper_dir;
+if ($opt_socket_wrapper)
{
- $socket_wrapper_dir = "$prefix/w";
- $ENV{SOCKET_WRAPPER_DIR} = $socket_wrapper_dir;
- print "SOCKET_WRAPPER_DIR=$ENV{SOCKET_WRAPPER_DIR}\n";
-} else {
- print "NOT USING SOCKET_WRAPPER\n";
+ $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
+ print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
}
# Start slapd before smbd
if ($ldap) {
- slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+ Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
+
print "LDAP PROVISIONING...";
- system("$bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}") or
- die("LDAP PROVISIONING failed: $bindir/smbscript $setupdir/provision $ENV{PROVISION_OPTIONS} --ldap-backend=$ENV{LDAPI}");
+ Samba4::provision_ldap($bindir, $setupdir);
# LDAP is slow
$torture_maxtime *= 2;
}
+if (defined($opt_expected_failures)) {
+ open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
+ while (<KNOWN>) {
+ chomp;
+ s/([ \t]+)\#(.*)$//;
+ push (@expected_failures, $_); }
+ close(KNOWN);
+}
+
my $test_fifo = "$prefix/smbd_test.fifo";
$ENV{SMBD_TEST_FIFO} = $test_fifo;
$ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 1;
+SocketWrapper::set_default_iface(1);
my $max_time = 5400;
if (defined($ENV{SMBD_MAX_TIME})) {
$max_time = $ENV{SMBD_MAX_TIME};
}
-smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG}, $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
+Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG},
+ $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
+
+SocketWrapper::set_default_iface(6);
-$ENV{SOCKET_WRAPPER_DEFAULT_IFACE} = 6;
-$ENV{TORTURE_INTERFACES} = '127.0.0.6/8,127.0.0.7/8,127.0.0.8/8,127.0.0.9/8,127.0.0.10/8,127.0.0.11/8';
+my $interfaces = join(',', ("127.0.0.6/8",
+ "127.0.0.7/8",
+ "127.0.0.8/8",
+ "127.0.0.9/8",
+ "127.0.0.10/8",
+ "127.0.0.11/8"));
-my @torture_options = ("--option=interfaces=$ENV{TORTURE_INTERFACES} $ENV{CONFIGURATION}");
+push (@torture_options, "--option=interfaces=$interfaces");
+push (@torture_options, $ENV{CONFIGURATION});
# ensure any one smbtorture call doesn't run too long
push (@torture_options, "--maximum-runtime=$torture_maxtime");
push (@torture_options, "--target=$opt_target");
-push (@torture_options, "--option=torture:progress=no")
- if (defined($ENV{RUN_FROM_BUILD_FARM}) and $ENV{RUN_FROM_BUILD_FARM} eq "yes");
+push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
+push (@torture_options, "--format=subunit");
+push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
$ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
-my $start = time();
-
open(DATA, ">$test_fifo");
-# give time for nbt server to register its names
-print "delaying for nbt name registration\n";
-sleep(4);
-
-# This will return quickly when things are up, but be slow if we need to wait for (eg) SSL init
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{SERVER}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} $ENV{NETBIOSNAME}");
-system("bin/nmblookup $ENV{CONFIGURATION} -U $ENV{SERVER} $ENV{NETBIOSNAME}");
-
-# start off with 0 failures
-$ENV{failed} = 0;
-my $totalfailed = 0;
-
my @todo = ();
if ($opt_target eq "win") {
$name =~ s/\n//g;
my $cmdline = <IN>;
$cmdline =~ s/\n//g;
- push (@todo, [$name, $cmdline]);
+ push (@todo, [$name, $cmdline])
+ if (not defined($tests) or $name =~ /$tests/);
} else {
print;
}
}
- close(IN);
+ close(IN) or die("Error creating recipe");
}
-my $total = $#todo + 1;
+Samba4::wait_for_start();
+
+# start off with 0 failures
+$ENV{failed} = 0;
+
+my $suitestotal = $#todo + 1;
my $i = 0;
$| = 1;
+delete $ENV{DOMAIN};
+
foreach (@todo) {
- $i = $i + 1;
- my $err = "";
- if ($totalfailed > 0) { $err = ", $totalfailed errors"; }
- printf "[$i/$total in " . (time() - $start)."s$err] $$_[0]\n";
- my $ret = system("$$_[1] >/dev/null 2>/dev/null");
- if ($ret != 0) {
- $totalfailed++;
- exit(1) if ($opt_one);
+ $i++;
+ my $cmd = $$_[1];
+ $cmd =~ s/([\(\)])/\\$1/g;
+ my $name = $$_[0];
+ if ($from_build_farm) {
+ run_test_buildfarm($name, $cmd, $i, $suitestotal);
+ } else {
+ run_test_plain($name, $cmd, $i, $suitestotal);
}
}
close(DATA);
+sleep(2);
+
my $failed = $? >> 8;
if (-f "$ENV{PIDDIR}/smbd.pid" ) {
close(IN);
}
-if ($ldap) {
- open(IN, "<$ENV{PIDDIR}/slapd.pid") or die("unable to open slapd pid file");
- kill 9, <IN>;
- close(IN);
-}
+Samba4::slapd_stop() if ($ldap);
-my $end=time();
-print "DURATION: " . ($end-$start). " seconds\n";
-print "$totalfailed failures\n";
+my $end = time();
+my $duration = ($end-$start);
+my $numfailed = $#$suitesfailed+1;
+if ($numfailed == 0) {
+ my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
+ print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
+} else {
+
+ unless ($from_build_farm) {
+ if (not $opt_immediate and not $opt_verbose) {
+ foreach (@$suitesfailed) {
+ print "===============================================================================\n";
+ print "FAIL: $_\n";
+ print $test_output->{$_};
+ print "\n";
+ }
+ }
+
+ print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
+ } else {
+ print <<EOF
+************************
+*** TESTSUITE FAILED ***
+************************
+EOF
+;
+ }
+}
+print "DURATION: $duration seconds\n";
# if there were any valgrind failures, show them
foreach (<$prefix/valgrind.log*>) {
}
}
-teststatus($Script, $failed);
-
-exit $failed;
+exit $numfailed;
git.samba.org / ab / samba.git / .git / blobdiff