#!/usr/bin/perl
# Bootstrap Samba and run a number of tests against it.
-# Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2005-2010 Jelmer Vernooij <jelmer@samba.org>
# Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
# This program is free software; you can redistribute it and/or modify
selftest --help
-selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
+selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
=head1 DESCRIPTION
Change directory to run tests in. Default is 'st'.
-=item I<--immediate>
-
-Show errors as soon as they happen rather than at the end of the test run.
-
=item I<--target samba4|samba3|win|kvm>
Specify test target against which to run. Default is 'samba4'.
use FindBin qw($RealBin $Script);
use File::Spec;
+use File::Temp qw(tempfile);
use Getopt::Long;
use POSIX;
use Cwd qw(abs_path);
use lib "$RealBin";
-use Subunit qw(parse_results);
+use Subunit;
use SocketWrapper;
+eval {
+require Time::HiRes;
+Time::HiRes->import("time");
+};
+if ($@) {
+ print "You don't have Time::Hires installed !\n";
+}
+
my $opt_help = 0;
my $opt_target = "samba4";
my $opt_quick = 0;
my $opt_socket_wrapper_pcap = undef;
my $opt_socket_wrapper_keep_pcap = undef;
my $opt_one = 0;
-my $opt_immediate = 0;
my @opt_exclude = ();
my @opt_include = ();
my $opt_verbose = 0;
my $opt_image = undef;
my $opt_testenv = 0;
+my $opt_list = 0;
my $ldap = undef;
-my $opt_analyse_cmd = undef;
my $opt_resetup_env = undef;
my $opt_bindir = undef;
-my $opt_no_lazy_setup = undef;
-my $opt_format = "plain";
+my $opt_load_list = undef;
my @testlists = ();
my $srcdir = ".";
my @includes = ();
my @excludes = ();
-my $statistics = {
- SUITES_FAIL => 0,
+sub pipe_handler {
+ my $sig = shift @_;
+ print STDERR "Exiting early because of SIGPIPE.\n";
+ exit(1);
+}
- TESTS_UNEXPECTED_OK => 0,
- TESTS_EXPECTED_OK => 0,
- TESTS_UNEXPECTED_FAIL => 0,
- TESTS_EXPECTED_FAIL => 0,
- TESTS_ERROR => 0,
- TESTS_SKIP => 0,
-};
+$SIG{PIPE} = \&pipe_handler;
sub find_in_list($$)
{
foreach (@$list) {
if ($fullname =~ /$$_[0]/) {
return ($$_[1]) if ($$_[1]);
- return "NO REASON SPECIFIED";
+ return "";
}
}
return $pcap_file;
}
-sub cleanup_pcap($$$)
+sub cleanup_pcap($$)
{
- my ($pcap_file, $expected_ret, $ret) = @_;
+ my ($pcap_file, $exitcode) = @_;
return unless ($opt_socket_wrapper_pcap);
return if ($opt_socket_wrapper_keep_pcap);
- return unless ($expected_ret == $ret);
+ return unless ($exitcode == 0);
return unless defined($pcap_file);
unlink($pcap_file);
}
-sub run_testsuite($$$$$$)
+# expand strings from %ENV
+sub expand_environment_strings($)
{
- my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
- my $pcap_file = setup_pcap($name);
+ my $s = shift;
+ # we use a reverse sort so we do the longer ones first
+ foreach my $k (sort { $b cmp $a } keys %ENV) {
+ $s =~ s/\$$k/$ENV{$k}/g;
+ }
+ return $s;
+}
- $msg_ops->report_time(time());
- $msg_ops->start_test([], $name);
+sub run_testsuite($$$$$)
+{
+ my ($envname, $name, $cmd, $i, $totalsuites) = @_;
+ my $pcap_file = setup_pcap($name);
- unless (open(RESULT, "$cmd 2>&1|")) {
- $statistics->{TESTS_ERROR}++;
- $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
- $statistics->{SUITES_FAIL}++;
- return 0;
+ Subunit::start_testsuite($name);
+ Subunit::progress_push();
+ Subunit::report_time(time());
+ system($cmd);
+ Subunit::report_time(time());
+ Subunit::progress_pop();
+
+ if ($? == -1) {
+ Subunit::progress_pop();
+ Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
+ exit(1);
+ } elsif ($? & 127) {
+ Subunit::end_testsuite($name, "error",
+ sprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127), ($? & 128) ? 'with' : 'without'));
+ exit(1);
}
- my $expected_ret = parse_results(
- $msg_ops, $statistics, *RESULT, [$name]);
+ my $exitcode = $? >> 8;
my $envlog = getlog_env($envname);
- $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
-
- $msg_ops->output_msg("CMD: $cmd\n");
-
- my $ret = close(RESULT);
- $ret = 0 unless $ret == 1;
+ if ($envlog ne "") {
+ print "envlog: $envlog\n";
+ }
- my $exitcode = $? >> 8;
+ print "command: $cmd\n";
+ printf "expanded command: %s\n", expand_environment_strings($cmd);
- $msg_ops->report_time(time());
- if ($ret == 1) {
- $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
+ if ($exitcode == 0) {
+ Subunit::end_testsuite($name, "success");
} else {
- $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
+ Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
}
- cleanup_pcap($pcap_file, $expected_ret, $ret);
+ cleanup_pcap($pcap_file, $exitcode);
if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
- $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
+ print "PCAP FILE: $pcap_file\n";
}
- if ($ret != $expected_ret) {
- $statistics->{SUITES_FAIL}++;
+ if ($exitcode != 0) {
exit(1) if ($opt_one);
}
- return ($ret == $expected_ret);
+ return $exitcode;
}
sub ShowHelp()
Behaviour:
--quick run quick overall test
--one abort when the first test fails
- --immediate print test output for failed tests during run
--verbose be verbose
- --analyse-cmd CMD command to run after each test
+ --testenv run a shell in the requested test environment
+ --list list available tests
";
exit(0);
}
'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
'quick' => \$opt_quick,
'one' => \$opt_one,
- 'immediate' => \$opt_immediate,
'exclude=s' => \@opt_exclude,
'include=s' => \@opt_include,
'srcdir=s' => \$srcdir,
'exeext=s' => \$exeext,
'verbose' => \$opt_verbose,
'testenv' => \$opt_testenv,
+ 'list' => \$opt_list,
'ldap:s' => \$ldap,
- 'analyse-cmd=s' => \$opt_analyse_cmd,
- 'no-lazy-setup' => \$opt_no_lazy_setup,
'resetup-environment' => \$opt_resetup_env,
'bindir:s' => \$opt_bindir,
- 'format=s' => \$opt_format,
'image=s' => \$opt_image,
- 'testlist=s' => \@testlists
+ 'testlist=s' => \@testlists,
+ 'load-list=s' => \$opt_load_list,
);
exit(1) if (not $result);
ShowHelp() if ($opt_help);
-my $tests = shift;
+die("--list and --testenv are mutually exclusive") if ($opt_list and $opt_testenv);
+
+# we want unbuffered output
+$| = 1;
+
+my @tests = @ARGV;
# quick hack to disable rpc validation when using valgrind - its way too slow
unless (defined($ENV{VALGRIND})) {
$ENV{MALLOC_CHECK_} = 2;
}
+# make all our python scripts unbuffered
+$ENV{PYTHONUNBUFFERED} = 1;
+
my $bindir = ($opt_bindir or "$builddir/bin");
my $bindir_abs = abs_path($bindir);
mkdir($prefix, 0777) unless -d $prefix;
my $prefix_abs = abs_path($prefix);
+my $tmpdir_abs = abs_path("$prefix/tmp");
+mkdir($tmpdir_abs, 0777) unless -d $tmpdir_abs;
+
my $srcdir_abs = abs_path($srcdir);
my $builddir_abs = abs_path($builddir);
$ENV{BUILDDIR_ABS} = $builddir_abs;
$ENV{EXEEXT} = $exeext;
-if (defined($ENV{RUN_FROM_BUILD_FARM}) and
- ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
- $opt_format = "buildfarm";
-}
-
my $tls_enabled = not $opt_quick;
$ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
-$ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
-$ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
+
sub prefix_pathvar($$)
{
my ($name, $newpath) = @_;
my $socket_wrapper_dir;
if ($opt_socket_wrapper) {
- $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
+ $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
} else {
unless ($< == 0) {
my $testenv_default = "none";
if ($opt_target eq "samba4") {
- $testenv_default = "member";
+ $testenv_default = "all";
require target::Samba4;
$target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
} elsif ($opt_target eq "samba3") {
push (@includes, read_test_regexes($_));
}
-my $interfaces = join(',', ("127.0.0.6/8",
- "127.0.0.7/8",
- "127.0.0.8/8",
- "127.0.0.9/8",
- "127.0.0.10/8",
- "127.0.0.11/8"));
+my $interfaces = join(',', ("127.0.0.11/8",
+ "127.0.0.12/8",
+ "127.0.0.13/8",
+ "127.0.0.14/8",
+ "127.0.0.15/8",
+ "127.0.0.16/8"));
+
+my $clientdir = "$prefix_abs/client";
-my $conffile = "$prefix_abs/client/client.conf";
+my $conffile = "$clientdir/client.conf";
$ENV{SMB_CONF_PATH} = $conffile;
-sub write_clientconf($$)
+sub write_clientconf($$$)
{
- my ($conffile, $vars) = @_;
+ my ($conffile, $clientdir, $vars) = @_;
+
+ mkdir("$clientdir", 0777) unless -d "$clientdir";
+
+ if ( -d "$clientdir/private" ) {
+ unlink <$clientdir/private/*>;
+ } else {
+ mkdir("$clientdir/private", 0777);
+ }
- mkdir("$prefix/client", 0777) unless -d "$prefix/client";
-
- if ( -d "$prefix/client/private" ) {
- unlink <$prefix/client/private/*>;
+ if ( -d "$clientdir/lockdir" ) {
+ unlink <$clientdir/lockdir/*>;
} else {
- mkdir("$prefix/client/private", 0777);
+ mkdir("$clientdir/lockdir", 0777);
}
- if ( -d "$prefix/client/lock" ) {
- unlink <$prefix/client/lockdir/*>;
+ if ( -d "$clientdir/ncalrpcdir" ) {
+ unlink <$clientdir/ncalrpcdir/*>;
} else {
- mkdir("$prefix/client/lockdir", 0777);
+ mkdir("$clientdir/ncalrpcdir", 0777);
}
open(CF, ">$conffile");
print CF "\tinterfaces = $interfaces\n";
}
print CF "
- private dir = $prefix_abs/client/private
- lock dir = $prefix_abs/client/lockdir
- name resolve order = bcast
+ private dir = $clientdir/private
+ lock dir = $clientdir/lockdir
+ ncalrpc dir = $clientdir/ncalrpcdir
+ name resolve order = bcast file
panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
max xmit = 32K
notify:inotify = false
ldb:nosync = true
system:anonymous = true
client lanman auth = Yes
- torture:basedir = $prefix_abs/client
+ log level = 1
+ torture:basedir = $clientdir
#We don't want to pass our self-tests if the PAC code is wrong
gensec:require_pac = true
- modules dir = $ENV{LD_SAMBA_MODULE_PATH}
+ setup directory = ./setup
+ resolv:host file = $prefix_abs/dns_host_file
+#We don't want to run 'speed' tests for very long
+ torture:timelimit = 1
";
close(CF);
}
my $testsdir = "$srcdir/selftest";
-my %required_envs = ();
+sub should_run_test($)
+{
+ my $name = shift;
+ if ($#tests == -1) {
+ return 1;
+ }
+ for (my $i=0; $i <= $#tests; $i++) {
+ if ($name =~ /$tests[$i]/i) {
+ return 1;
+ }
+ }
+ return 0;
+}
sub read_testlist($)
{
open(IN, $filename) or die("Unable to open $filename: $!");
while (<IN>) {
- if ($_ eq "-- TEST --\n") {
+ if (/-- TEST(-LOADLIST|-IDLIST|) --\n/) {
+ my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
+ my $supports_idlist = (defined($1) and $1 eq "-IDLIST");
my $name = <IN>;
$name =~ s/\n//g;
my $env = <IN>;
$env =~ s/\n//g;
my $cmdline = <IN>;
$cmdline =~ s/\n//g;
- if (not defined($tests) or $name =~ /$tests/) {
- $required_envs{$env} = 1;
- push (@ret, [$name, $env, $cmdline]);
+ if (should_run_test($name) == 1) {
+ push (@ret, [$name, $env, $cmdline, $supports_loadlist, $supports_idlist]);
}
} else {
print;
}
$ENV{SELFTEST_PREFIX} = "$prefix_abs";
+$ENV{SELFTEST_TMPDIR} = "$tmpdir_abs";
+$ENV{TEST_DATA_PREFIX} = "$tmpdir_abs";
if ($opt_socket_wrapper) {
$ENV{SELFTEST_INTERFACES} = $interfaces;
} else {
foreach my $fn (@testlists) {
foreach (read_testlist($fn)) {
my $name = $$_[0];
- next if (@includes and not find_in_list(\@includes, $name));
+ next if (@includes and not defined(find_in_list(\@includes, $name)));
push (@available, $_);
}
}
-my $msg_ops;
-if ($opt_format eq "buildfarm") {
- require output::buildfarm;
- $msg_ops = new output::buildfarm($statistics);
-} elsif ($opt_format eq "plain") {
- require output::plain;
- $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
-} elsif ($opt_format eq "html") {
- require output::html;
- mkdir("test-results", 0777);
- $msg_ops = new output::html("test-results", $statistics);
-} elsif ($opt_format eq "subunit") {
- require output::subunit;
- $msg_ops = new output::subunit();
-} else {
- die("Invalid output format '$opt_format'");
+my $restricted = undef;
+my $restricted_used = {};
+
+if ($opt_load_list) {
+ $restricted = [];
+ open(LOAD_LIST, "<$opt_load_list") or die("Unable to open $opt_load_list");
+ while (<LOAD_LIST>) {
+ chomp;
+ push (@$restricted, $_);
+ }
+ close(LOAD_LIST);
}
-$msg_ops->report_time(time());
-foreach (@available) {
- my $name = $$_[0];
+my $individual_tests = undef;
+$individual_tests = {};
+
+foreach my $testsuite (@available) {
+ my $name = $$testsuite[0];
my $skipreason = skip($name);
- if ($skipreason) {
- $msg_ops->skip_testsuite($name, $skipreason);
+ if (defined($restricted)) {
+ # Find the testsuite for this test
+ my $match = undef;
+ foreach my $r (@$restricted) {
+ if ($r eq $name) {
+ $individual_tests->{$name} = [];
+ $match = $r;
+ $restricted_used->{$r} = 1;
+ } elsif (substr($r, 0, length($name)+1) eq "$name.") {
+ push(@{$individual_tests->{$name}}, $r);
+ $match = $r;
+ $restricted_used->{$r} = 1;
+ }
+ }
+ if ($match) {
+ if (defined($skipreason)) {
+ Subunit::skip_testsuite($name, $skipreason);
+ } else {
+ push(@todo, $testsuite);
+ }
+ }
+ } elsif (defined($skipreason)) {
+ Subunit::skip_testsuite($name, $skipreason);
} else {
- push(@todo, $_);
+ push(@todo, $testsuite);
}
}
-if ($#todo == -1) {
+if (defined($restricted)) {
+ foreach (@$restricted) {
+ unless (defined($restricted_used->{$_})) {
+ print "No test or testsuite found matching $_\n";
+ }
+ }
+} elsif ($#todo == -1) {
print STDERR "No tests to run\n";
exit(1);
- }
+}
my $suitestotal = $#todo + 1;
+
+Subunit::progress($suitestotal);
+Subunit::report_time(time());
+
my $i = 0;
$| = 1;
"DC_NETBIOSNAME",
"DC_NETBIOSALIAS",
+ # domain member
+ "MEMBER_SERVER",
+ "MEMBER_SERVER_IP",
+ "MEMBER_NETBIOSNAME",
+ "MEMBER_NETBIOSALIAS",
+
+ # rpc proxy controller stuff
+ "RPC_PROXY_SERVER",
+ "RPC_PROXY_SERVER_IP",
+ "RPC_PROXY_NETBIOSNAME",
+ "RPC_PROXY_NETBIOSALIAS",
+
+ # domain controller stuff for Vampired DC
+ "VAMPIRE_DC_SERVER",
+ "VAMPIRE_DC_SERVER_IP",
+ "VAMPIRE_DC_NETBIOSNAME",
+ "VAMPIRE_DC_NETBIOSALIAS",
+
# server stuff
"SERVER",
"SERVER_IP",
# user stuff
"USERNAME",
+ "USERID",
"PASSWORD",
"DC_USERNAME",
"DC_PASSWORD",
# misc stuff
"KRB5_CONFIG",
"WINBINDD_SOCKET_DIR",
- "WINBINDD_PRIV_PIPE_DIR"
+ "WINBINDD_PRIV_PIPE_DIR",
+ "NMBD_SOCKET_DIR",
+ "LOCAL_PATH"
);
$SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
die("Received signal $signame");
};
-sub setup_env($)
+sub setup_env($$)
{
- my ($name) = @_;
+ my ($name, $prefix) = @_;
my $testenv_vars = undef;
} elsif (defined(get_running_env($envname))) {
$testenv_vars = get_running_env($envname);
if (not $target->check_env($testenv_vars)) {
+ print $target->getlog_env($testenv_vars);
$testenv_vars = undef;
}
} else {
SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
$ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
} elsif ($option eq "client") {
- SocketWrapper::set_default_iface(6);
- write_clientconf($conffile, $testenv_vars);
+ SocketWrapper::set_default_iface(11);
+ write_clientconf($conffile, $clientdir, $testenv_vars);
$ENV{SMB_CONF_PATH} = $conffile;
} else {
die("Unknown option[$option] for envname[$envname]");
delete $running_envs{$envname};
}
-if ($opt_no_lazy_setup) {
- setup_env($_) foreach (keys %required_envs);
-}
+# This 'global' file needs to be empty when we start
+unlink("$prefix_abs/dns_host_file");
if ($opt_testenv) {
my $testenv_name = $ENV{SELFTEST_TESTENV};
$testenv_name = $testenv_default unless defined($testenv_name);
- my $testenv_vars = setup_env($testenv_name);
+ my $testenv_vars = setup_env($testenv_name, $prefix);
+
+ die("Unable to setup environment $testenv_name") unless ($testenv_vars);
$ENV{PIDDIR} = $testenv_vars->{PIDDIR};
+ $ENV{ENVNAME} = $testenv_name;
my $envvarstr = exported_envvars_str($testenv_vars);
- my $term = ($ENV{TERM} or "xterm");
- system("$term -e 'echo -e \"
+ my $term = ($ENV{TERMINAL} or "xterm -e");
+ system("$term 'echo -e \"
Welcome to the Samba4 Test environment '$testenv_name'
This matches the client environment used in make test
$envvarstr
\" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
teardown_env($testenv_name);
+} elsif ($opt_list) {
+ foreach (@todo) {
+ my $cmd = $$_[2];
+ my $name = $$_[0];
+ my $envname = $$_[1];
+
+ unless($cmd =~ /\$LISTOPT/) {
+ warn("Unable to list tests in $name");
+ next;
+ }
+
+ $cmd =~ s/\$LISTOPT/--list/g;
+
+ system($cmd);
+
+ if ($? == -1) {
+ die("Unable to run $cmd: $!");
+ } elsif ($? & 127) {
+ die(snprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127), ($? & 128) ? 'with' : 'without'));
+ }
+
+ my $exitcode = $? >> 8;
+ if ($exitcode != 0) {
+ die("$cmd exited with exit code $exitcode");
+ }
+ }
} else {
foreach (@todo) {
$i++;
my $cmd = $$_[2];
- $cmd =~ s/([\(\)])/\\$1/g;
my $name = $$_[0];
my $envname = $$_[1];
-
- my $envvars = setup_env($envname);
+
+ my $envvars = setup_env($envname, $prefix);
if (not defined($envvars)) {
- $msg_ops->skip_testsuite($name,
- "unable to set up environment $envname");
+ Subunit::start_testsuite($name);
+ Subunit::end_testsuite($name, "error",
+ "unable to set up environment $envname - exiting");
next;
}
- run_testsuite($envname, $name, $cmd, $i, $suitestotal,
- $msg_ops);
-
- if (defined($opt_analyse_cmd)) {
- system("$opt_analyse_cmd \"$name\"");
+ # Generate a file with the individual tests to run, if the
+ # test runner for this test suite supports it.
+ if ($individual_tests and $individual_tests->{$name}) {
+ if ($$_[3]) {
+ my ($fh, $listid_file) = tempfile(UNLINK => 0);
+ foreach my $test (@{$individual_tests->{$name}}) {
+ print $fh substr($test, length($name)+1) . "\n";
+ }
+ $cmd =~ s/\$LOADLIST/--load-list=$listid_file/g;
+ } elsif ($$_[4]) {
+ $cmd =~ s/\s+[^\s]+\s*$//;
+ $cmd .= " " . join(' ', @{$individual_tests->{$name}});
+ }
}
+ run_testsuite($envname, $name, $cmd, $i, $suitestotal);
+
teardown_env($envname) if ($opt_resetup_env);
}
}
teardown_env($_) foreach (keys %running_envs);
-$target->stop();
-
-$msg_ops->summary();
-
my $failed = 0;
# if there were any valgrind failures, show them
foreach (<$prefix/valgrind.log*>) {
next unless (-s $_);
- system("grep DWARF2.CFI.reader $_ > /dev/null");
- if ($? >> 8 == 0) {
- print "VALGRIND FAILURE\n";
- $failed++;
- system("cat $_");
- }
+ print "VALGRIND FAILURE\n";
+ $failed++;
+ system("cat $_");
}
-
-if ($opt_format eq "buildfarm") {
- print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
-}
-
-exit $statistics->{SUITES_FAIL};
+exit 0;