2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME/TEST-NAME
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
79 Abort as soon as one test fails.
87 =item I<SMBD_VALGRIND>
89 =item I<TORTURE_MAXTIME>
101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
112 use FindBin qw($RealBin $Script);
116 use Cwd qw(abs_path);
123 my $opt_target = "samba4";
125 my $opt_socket_wrapper = 0;
126 my $opt_socket_wrapper_pcap = undef;
127 my $opt_socket_wrapper_keep_pcap = undef;
129 my $opt_immediate = 0;
130 my $opt_expected_failures = undef;
131 my $opt_skip = undef;
135 my $opt_analyse_cmd = undef;
136 my $opt_resetup_env = undef;
137 my $opt_bindir = undef;
143 my $suitesfailed = [];
145 my @expected_failures = ();
153 TESTS_UNEXPECTED_OK => 0,
154 TESTS_EXPECTED_OK => 0,
155 TESTS_UNEXPECTED_FAIL => 0,
156 TESTS_EXPECTED_FAIL => 0,
160 sub expecting_failure($)
162 my $fullname = shift;
164 return 1 if (grep(/^$fullname$/, @expected_failures));
171 my $fullname = shift;
173 return 1 if (grep(/^$fullname$/, @skips));
177 my $test_output = {};
179 sub buildfarm_start_msg($)
184 $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
185 $out .= "Running test $state->{NAME} (level 0 stdout)\n";
186 $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
187 $out .= scalar(localtime())."\n";
188 $out .= "NAME: $state->{NAME}\n";
189 $out .= "CMD: $state->{CMD}\n";
191 $test_output->{$state->{NAME}} = "";
196 sub buildfarm_output_msg($$)
198 my ($state, $output) = @_;
200 $test_output->{$state->{NAME}} .= $output;
203 sub buildfarm_end_msg($$$)
205 my ($state, $expected_ret, $ret) = @_;
208 if ($ret == $expected_ret) {
211 $out .= "ERROR: $ret";
212 $out .= $test_output->{$state->{NAME}};
215 $out .= "==========================================\n";
216 if ($ret == $expected_ret) {
217 $out .= "TEST PASSED: $state->{NAME}\n";
219 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
221 $out .= "==========================================\n";
226 my $buildfarm_msg_ops = {
227 start_msg => \&buildfarm_start_msg,
228 output_msg => \&buildfarm_output_msg,
229 end_msg => \&buildfarm_end_msg
232 sub plain_output_msg($$);
234 sub plain_start_msg($)
239 $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
240 $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
241 $out .= "] $state->{NAME}\n";
243 $test_output->{$state->{NAME}} = "" unless $opt_verbose;
245 plain_output_msg($state, "CMD: $state->{CMD}\n");
250 sub plain_output_msg($$)
252 my ($state, $output) = @_;
257 $test_output->{$state->{NAME}} .= $output;
261 sub plain_end_msg($$$)
263 my ($state, $expected_ret, $ret) = @_;
265 if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
266 print $test_output->{$state->{NAME}}."\n";
270 my $plain_msg_ops = {
271 start_msg => \&plain_start_msg,
272 output_msg => \&plain_output_msg,
273 end_msg => \&plain_end_msg
278 my ($name, $cmd, $i, $totalsuites, $msg_ops) = @_;
283 TOTAL => $totalsuites,
286 $msg_ops->{start_msg}($msg_state);
287 open(RESULT, "$cmd 2>&1|");
288 my $expected_ret = 1;
291 $msg_ops->{output_msg}($msg_state, $_);
292 if (/^test: (.+)\n/) {
293 $open_tests->{$1} = 1;
294 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
296 if ($1 eq "success") {
297 delete $open_tests->{$2};
298 if (expecting_failure("$name/$2")) {
299 $statistics->{TESTS_UNEXPECTED_OK}++;
301 $statistics->{TESTS_EXPECTED_OK}++;
303 } elsif ($1 eq "failure") {
304 delete $open_tests->{$2};
305 if (expecting_failure("$name/$2")) {
306 $statistics->{TESTS_EXPECTED_FAIL}++;
309 $statistics->{TESTS_UNEXPECTED_FAIL}++;
311 } elsif ($1 eq "skip") {
312 delete $open_tests->{$2};
313 } elsif ($1 eq "error") {
314 $statistics->{TESTS_ERROR}++;
315 delete $open_tests->{$2};
319 foreach (keys %$open_tests) {
320 $msg_ops->{output_msg}($msg_state, "$_ was started but never finished!\n");
321 $statistics->{TESTS_ERROR}++;
323 my $ret = close(RESULT);
324 $msg_ops->{end_msg}($msg_state, $expected_ret, $ret);
325 if ($ret != $expected_ret) {
326 push(@$suitesfailed, $name);
327 $statistics->{SUITES_FAIL}++;
328 exit(1) if ($opt_one);
330 $statistics->{SUITES_OK}++;
333 return ($ret == $expected_ret);
338 print "Samba test runner
339 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
341 Usage: $Script [OPTIONS] PREFIX
344 --help this help page
345 --target=samba4|samba3|win Samba version to target
348 --prefix=DIR prefix to run tests in [st]
349 --srcdir=DIR source directory [.]
350 --builddir=DIR output directory [.]
353 --socket-wrapper-pcap=DIR save traffic to pcap directories
354 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
356 --socket-wrapper enable socket wrapper
357 --expected-failures=FILE specify list of tests that is guaranteed to fail
360 --ldap=openldap|fedora back smbd onto specified ldap server
363 --bindir=PATH path to binaries
366 --quick run quick overall test
367 --one abort when the first test fails
368 --immediate print test output for failed tests during run
370 --analyse-cmd CMD command to run after each test
375 my $result = GetOptions (
376 'help|h|?' => \$opt_help,
377 'target=s' => \$opt_target,
378 'prefix=s' => \$prefix,
379 'socket-wrapper' => \$opt_socket_wrapper,
380 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
381 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
382 'quick' => \$opt_quick,
384 'immediate' => \$opt_immediate,
385 'expected-failures=s' => \$opt_expected_failures,
386 'skip=s' => \$opt_skip,
387 'srcdir=s' => \$srcdir,
388 'builddir=s' => \$builddir,
389 'verbose' => \$opt_verbose,
390 'testenv' => \$opt_testenv,
392 'analyse-cmd=s' => \$opt_analyse_cmd,
393 'resetup-environment' => \$opt_resetup_env,
394 'bindir:s' => \$opt_bindir,
397 exit(1) if (not $result);
399 ShowHelp() if ($opt_help);
403 # quick hack to disable rpc validation when using valgrind - its way too slow
404 unless (defined($ENV{VALGRIND})) {
405 $ENV{VALIDATE} = "validate";
406 $ENV{MALLOC_CHECK_} = 2;
409 my $old_pwd = "$RealBin/../..";
411 # Backwards compatibility:
412 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
413 if (defined($ENV{FEDORA_DS_PREFIX})) {
420 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
423 $torture_maxtime *= 2;
430 my $prefix_abs = abs_path($prefix);
431 my $srcdir_abs = abs_path($srcdir);
433 die("using an empty prefix isn't allowed") unless $prefix ne "";
434 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
435 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
437 $ENV{PREFIX} = $prefix;
438 $ENV{SRCDIR} = $srcdir;
440 #Ensure we have the test prefix around
441 mkdir $prefix unless -d $prefix;
443 my $tls_enabled = not $opt_quick;
444 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
445 ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
447 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
448 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
449 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
450 if (defined($ENV{LD_LIBRARY_PATH})) {
451 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
453 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
455 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
456 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
458 my $pcap_dir = "$prefix/pcap";
460 if ($opt_socket_wrapper_pcap) {
462 # Socket wrapper pcap implies socket wrapper
463 $opt_socket_wrapper = 1;
466 my $socket_wrapper_dir;
467 if ($opt_socket_wrapper) {
468 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
469 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
471 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
476 if ($opt_target eq "samba4") {
477 $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
478 } elsif ($opt_target eq "samba3") {
479 if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
480 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
483 $target = new Samba3($opt_bindir);
484 } elsif ($opt_target eq "win") {
485 die("Windows tests will not run with socket wrapper enabled.")
486 if ($opt_socket_wrapper);
487 $target = new Windows();
490 if (defined($opt_expected_failures)) {
491 open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
495 push (@expected_failures, $_); }
499 if (defined($opt_skip)) {
500 open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
508 my $interfaces = join(',', ("127.0.0.6/8",
515 my $conffile = "$prefix_abs/client/client.conf";
517 sub write_clientconf($$)
519 my ($conffile, $vars) = @_;
521 mkdir "$prefix/client" unless -d "$prefix/client";
523 if ( -d "$prefix/client/private" ) {
524 unlink <$prefix/client/private/*>;
526 mkdir("$prefix/client/private");
529 open(CF, ">$conffile");
530 print CF "[global]\n";
531 if (defined($ENV{VALGRIND})) {
532 print CF "\ticonv:native = true\n";
534 print CF "\ticonv:native = false\n";
536 print CF "\tnetbios name = client\n";
537 if (defined($vars->{DOMAIN})) {
538 print CF "\tworkgroup = $vars->{DOMAIN}\n";
540 if (defined($vars->{REALM})) {
541 print CF "\trealm = $vars->{REALM}\n";
543 if (defined($vars->{NCALRPCDIR})) {
544 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
546 if (defined($vars->{PIDDIR})) {
547 print CF "\tpid directory = $vars->{PIDDIR}\n";
549 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
550 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
553 private dir = $srcdir_abs/$prefix/client/private
554 js include = $srcdir_abs/scripting/libjs
555 name resolve order = bcast
556 interfaces = $interfaces
557 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
559 notify:inotify = false
561 system:anonymous = true
562 torture:basedir = $prefix_abs
563 #We don't want to pass our self-tests if the PAC code is wrong
564 gensec:require_pac = true
570 my @torture_options = ();
571 push (@torture_options, "--configfile=$conffile");
572 # ensure any one smbtorture call doesn't run too long
573 push (@torture_options, "--maximum-runtime=$torture_maxtime");
574 push (@torture_options, "--target=$opt_target");
575 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
576 push (@torture_options, "--format=subunit");
577 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
579 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
580 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
584 my $testsdir = "$srcdir/script/tests";
585 $ENV{CONFIGURATION} = "--configfile=$conffile";
589 open(IN, "$testsdir/tests_quick.sh|");
591 open(IN, "$testsdir/tests_all.sh|");
594 if ($_ eq "-- TEST --\n") {
601 push (@todo, [$name, $env, $cmdline])
602 if (not defined($tests) or $name =~ /$tests/);
607 close(IN) or die("Error creating recipe");
609 my $suitestotal = $#todo + 1;
613 my %running_envs = ();
620 if (defined($running_envs{$envname})) {
621 $testenv_vars = $running_envs{$envname};
622 } elsif ($envname eq "none") {
625 $testenv_vars = $target->setup_env($envname, $prefix);
627 write_clientconf($conffile, $testenv_vars);
628 foreach ("PASSWORD", "DOMAIN", "SERVER", "USERNAME", "NETBIOSNAME",
629 "KRB5_CONFIG", "REALM") {
630 if (defined($testenv_vars->{$_})) {
631 $ENV{$_} = $testenv_vars->{$_};
637 SocketWrapper::set_default_iface(6);
639 $running_envs{$envname} = $testenv_vars;
640 return $testenv_vars;
646 return if ($envname eq "none");
647 $target->teardown_env($running_envs{$envname});
648 delete $running_envs{$envname};
653 my $testenv_vars = setup_env("dc");
654 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
655 my $term = ($ENV{TERM} or "xterm");
656 system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
657 This matches the client environment used in make test
658 smbd is pid `cat \$PIDDIR/smbd.pid`
660 Some useful environment variables:
661 TORTURE_OPTIONS=\$TORTURE_OPTIONS
662 CONFIGURATION=\$CONFIGURATION
664 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
670 $cmd =~ s/([\(\)])/\\$1/g;
672 my $envname = $$_[1];
675 print "SKIPPED: $name\n";
676 $statistics->{SUITES_SKIPPED}++;
683 $shname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
684 my $pcap_file = "$pcap_dir/$shname.cap";
686 SocketWrapper::setup_pcap($pcap_file) if ($opt_socket_wrapper_pcap);
688 if ($from_build_farm) {
689 $result = run_test($name, $cmd, $i, $suitestotal, $buildfarm_msg_ops);
691 $result = run_test($name, $cmd, $i, $suitestotal, $plain_msg_ops);
694 if ($opt_socket_wrapper_pcap and $result and
695 not $opt_socket_wrapper_keep_pcap) {
699 if (defined($opt_analyse_cmd)) {
700 system("$opt_analyse_cmd \"$name\"");
703 teardown_env($envname) if ($opt_resetup_env);
709 teardown_env($_) foreach (keys %running_envs);
714 my $duration = ($end-$start);
715 my $numfailed = $#$suitesfailed+1;
716 if ($numfailed == 0) {
717 my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
718 print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
720 unless ($from_build_farm) {
721 if (not $opt_immediate and not $opt_verbose) {
722 foreach (@$suitesfailed) {
723 print "===============================================================================\n";
725 print $test_output->{$_};
730 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
733 print "DURATION: $duration seconds\n";
737 # if there were any valgrind failures, show them
738 foreach (<$prefix/valgrind.log*>) {
740 system("grep DWARF2.CFI.reader $_ > /dev/null");
742 print "VALGRIND FAILURE\n";
748 if ($from_build_farm) {
749 print "TEST STATUS: $numfailed\n";