2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME/TEST-NAME
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
79 Abort as soon as one test fails.
87 =item I<SMBD_VALGRIND>
89 =item I<TORTURE_MAXTIME>
101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
111 use FindBin qw($RealBin $Script);
115 use Cwd qw(abs_path);
123 my $opt_target = "samba4";
125 my $opt_socket_wrapper = 0;
126 my $opt_socket_wrapper_pcap = undef;
127 my $opt_socket_wrapper_keep_pcap = undef;
129 my $opt_immediate = 0;
130 my $opt_expected_failures = undef;
131 my $opt_skip = undef;
135 my $opt_analyse_cmd = undef;
136 my $opt_resetup_env = undef;
137 my $opt_bindir = undef;
138 my $opt_no_lazy_setup = undef;
139 my $opt_format = "plain";
145 my @expected_failures = ();
149 START_TIME => time(),
155 TESTS_UNEXPECTED_OK => 0,
156 TESTS_EXPECTED_OK => 0,
157 TESTS_UNEXPECTED_FAIL => 0,
158 TESTS_EXPECTED_FAIL => 0,
162 sub expecting_failure($)
164 my $fullname = shift;
166 foreach (@expected_failures) {
167 return 1 if ($fullname =~ /$_/);
175 my $fullname = shift;
178 return 1 if ($fullname =~ /$_/);
190 return unless ($opt_socket_wrapper_pcap);
191 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
193 my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
194 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
196 $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
198 SocketWrapper::setup_pcap($state->{PCAP_FILE});
201 sub cleanup_pcap($$$)
203 my ($state, $expected_ret, $ret) = @_;
205 return unless ($opt_socket_wrapper_pcap);
206 return if ($opt_socket_wrapper_keep_pcap);
207 return unless ($expected_ret == $ret);
208 return unless defined($state->{PCAP_FILE});
210 unlink($state->{PCAP_FILE});
211 $state->{PCAP_FILE} = undef;
214 sub parse_subunit_results($$$$)
216 my ($msg_ops, $msg_state, $statistics, $fh) = @_;
217 my $expected_ret = 1;
221 $msg_ops->output_msg($msg_state, $_);
222 if (/^test: (.+)\n/) {
223 $open_tests->{$1} = 1;
224 $msg_ops->start_test($msg_state, $1);
225 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
227 if ($1 eq "success") {
228 delete $open_tests->{$2};
229 if (expecting_failure("$msg_state->{NAME}/$2")) {
230 $statistics->{TESTS_UNEXPECTED_OK}++;
231 $msg_ops->end_test($msg_state, $2, $1, 1);
233 $statistics->{TESTS_EXPECTED_OK}++;
234 $msg_ops->end_test($msg_state, $2, $1, 0);
236 } elsif ($1 eq "failure") {
237 delete $open_tests->{$2};
238 if (expecting_failure("$msg_state->{NAME}/$2")) {
239 $statistics->{TESTS_EXPECTED_FAIL}++;
240 $msg_ops->end_test($msg_state, $2, $1, 0);
243 $statistics->{TESTS_UNEXPECTED_FAIL}++;
244 $msg_ops->end_test($msg_state, $2, $1, 1);
246 } elsif ($1 eq "skip") {
247 delete $open_tests->{$2};
248 $msg_ops->end_test($msg_state, $2, $1, 0);
249 } elsif ($1 eq "error") {
250 $statistics->{TESTS_ERROR}++;
251 delete $open_tests->{$2};
252 $msg_ops->end_test($msg_state, $2, $1, 1);
257 foreach (keys %$open_tests) {
258 $msg_ops->end_test($msg_state, $_, "error", 1);
259 $msg_ops->output_msg($msg_state, "$_ was started but never finished!");
260 $statistics->{TESTS_ERROR}++;
263 return $expected_ret;
268 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
274 TOTAL => $totalsuites,
278 setup_pcap($msg_state);
280 open(RESULT, "$cmd 2>&1|");
281 $msg_ops->start_testsuite($msg_state);
283 my $expected_ret = parse_subunit_results(
284 $msg_ops, $msg_state, $statistics, *RESULT);
286 my $ret = close(RESULT);
288 cleanup_pcap($msg_state, $expected_ret, $ret);
290 $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
291 getlog_env($msg_state->{ENVNAME}));
293 if (not $opt_socket_wrapper_keep_pcap and
294 defined($msg_state->{PCAP_FILE})) {
295 $msg_ops->output_msg($msg_state,
296 "PCAP FILE: $msg_state->{PCAP_FILE}\n");
299 if ($ret != $expected_ret) {
300 $statistics->{SUITES_FAIL}++;
301 exit(1) if ($opt_one);
303 $statistics->{SUITES_OK}++;
306 return ($ret == $expected_ret);
311 print "Samba test runner
312 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
314 Usage: $Script [OPTIONS] PREFIX
317 --help this help page
318 --target=samba4|samba3|win Samba version to target
321 --prefix=DIR prefix to run tests in [st]
322 --srcdir=DIR source directory [.]
323 --builddir=DIR output directory [.]
326 --socket-wrapper-pcap=DIR save traffic to pcap directories
327 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
329 --socket-wrapper enable socket wrapper
330 --expected-failures=FILE specify list of tests that is guaranteed to fail
333 --ldap=openldap|fedora-ds back smbd onto specified ldap server
336 --bindir=PATH path to binaries
339 --quick run quick overall test
340 --one abort when the first test fails
341 --immediate print test output for failed tests during run
343 --analyse-cmd CMD command to run after each test
348 my $result = GetOptions (
349 'help|h|?' => \$opt_help,
350 'target=s' => \$opt_target,
351 'prefix=s' => \$prefix,
352 'socket-wrapper' => \$opt_socket_wrapper,
353 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
354 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
355 'quick' => \$opt_quick,
357 'immediate' => \$opt_immediate,
358 'expected-failures=s' => \$opt_expected_failures,
359 'skip=s' => \$opt_skip,
360 'srcdir=s' => \$srcdir,
361 'builddir=s' => \$builddir,
362 'verbose' => \$opt_verbose,
363 'testenv' => \$opt_testenv,
365 'analyse-cmd=s' => \$opt_analyse_cmd,
366 'no-lazy-setup' => \$opt_no_lazy_setup,
367 'resetup-environment' => \$opt_resetup_env,
368 'bindir:s' => \$opt_bindir,
369 'format=s' => \$opt_format,
372 exit(1) if (not $result);
374 ShowHelp() if ($opt_help);
378 # quick hack to disable rpc validation when using valgrind - its way too slow
379 unless (defined($ENV{VALGRIND})) {
380 $ENV{VALIDATE} = "validate";
381 $ENV{MALLOC_CHECK_} = 2;
384 my $old_pwd = "$RealBin/..";
386 # Backwards compatibility:
387 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
388 if (defined($ENV{FEDORA_DS_PREFIX})) {
395 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
398 $torture_maxtime *= 2;
405 die("using an empty prefix isn't allowed") unless $prefix ne "";
407 #Ensure we have the test prefix around
408 mkdir($prefix, 0777) unless -d $prefix;
410 my $prefix_abs = abs_path($prefix);
411 my $srcdir_abs = abs_path($srcdir);
413 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
414 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
416 $ENV{PREFIX} = $prefix;
417 $ENV{PREFIX_ABS} = $prefix_abs;
418 $ENV{SRCDIR} = $srcdir;
419 $ENV{SRCDIR_ABS} = $srcdir_abs;
421 my $tls_enabled = not $opt_quick;
422 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
423 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
424 $opt_format = "buildfarm";
427 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
428 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
429 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
430 if (defined($ENV{LD_LIBRARY_PATH})) {
431 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
433 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
435 if (defined($ENV{PKG_CONFIG_PATH})) {
436 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
438 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
440 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
443 if ($opt_socket_wrapper_pcap) {
444 # Socket wrapper pcap implies socket wrapper
445 $opt_socket_wrapper = 1;
448 my $socket_wrapper_dir;
449 if ($opt_socket_wrapper) {
450 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
451 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
453 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
458 if ($opt_target eq "samba4") {
459 $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
460 } elsif ($opt_target eq "samba3") {
461 if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
462 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
465 $target = new Samba3($opt_bindir);
466 } elsif ($opt_target eq "win") {
467 die("Windows tests will not run with socket wrapper enabled.")
468 if ($opt_socket_wrapper);
469 $target = new Windows();
472 if (defined($opt_expected_failures)) {
473 open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
477 push (@expected_failures, $_); }
481 if (defined($opt_skip)) {
482 open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
490 my $interfaces = join(',', ("127.0.0.6/8",
497 my $conffile = "$prefix_abs/client/client.conf";
499 sub write_clientconf($$)
501 my ($conffile, $vars) = @_;
503 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
505 if ( -d "$prefix/client/private" ) {
506 unlink <$prefix/client/private/*>;
508 mkdir("$prefix/client/private", 0777);
511 open(CF, ">$conffile");
512 print CF "[global]\n";
513 if (defined($ENV{VALGRIND})) {
514 print CF "\ticonv:native = true\n";
516 print CF "\ticonv:native = false\n";
518 print CF "\tnetbios name = client\n";
519 if (defined($vars->{DOMAIN})) {
520 print CF "\tworkgroup = $vars->{DOMAIN}\n";
522 if (defined($vars->{REALM})) {
523 print CF "\trealm = $vars->{REALM}\n";
525 if (defined($vars->{NCALRPCDIR})) {
526 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
528 if (defined($vars->{PIDDIR})) {
529 print CF "\tpid directory = $vars->{PIDDIR}\n";
531 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
532 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
535 private dir = $prefix_abs/client/private
536 js include = $srcdir_abs/scripting/libjs
537 name resolve order = bcast
538 interfaces = $interfaces
539 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
541 notify:inotify = false
543 system:anonymous = true
544 torture:basedir = $prefix_abs/client
545 #We don't want to pass our self-tests if the PAC code is wrong
546 gensec:require_pac = true
552 my @torture_options = ();
553 push (@torture_options, "--configfile=$conffile");
554 # ensure any one smbtorture call doesn't run too long
555 push (@torture_options, "--maximum-runtime=$torture_maxtime");
556 push (@torture_options, "--target=$opt_target");
557 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
558 push (@torture_options, "--format=subunit");
559 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
561 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
562 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
566 my $testsdir = "$srcdir/selftest";
567 $ENV{SMB_CONF_PATH} = "$conffile";
568 $ENV{CONFIGURATION} = "--configfile=$conffile";
570 my %required_envs = ();
573 open(IN, "$testsdir/tests_quick.sh|");
575 open(IN, "$testsdir/tests_all.sh|");
578 if ($_ eq "-- TEST --\n") {
585 if (not defined($tests) or $name =~ /$tests/) {
586 $required_envs{$env} = 1;
587 push (@todo, [$name, $env, $cmdline]);
593 close(IN) or die("Error creating recipe");
595 my $suitestotal = $#todo + 1;
599 my %running_envs = ();
601 my @exported_envvars = (
606 # domain controller stuff
626 "WINBINDD_SOCKET_DIR"
634 if ($envname eq "none") {
636 } elsif (defined($running_envs{$envname})) {
637 $testenv_vars = $running_envs{$envname};
638 if (not $target->check_env($testenv_vars)) {
639 $testenv_vars = undef;
642 $testenv_vars = $target->setup_env($envname, $prefix);
645 return undef unless defined($testenv_vars);
647 SocketWrapper::set_default_iface(6);
648 write_clientconf($conffile, $testenv_vars);
650 foreach (@exported_envvars) {
651 if (defined($testenv_vars->{$_})) {
652 $ENV{$_} = $testenv_vars->{$_};
658 $running_envs{$envname} = $testenv_vars;
659 return $testenv_vars;
662 sub exported_envvars_str($)
664 my ($testenv_vars) = @_;
667 foreach (@exported_envvars) {
668 next unless defined($testenv_vars->{$_});
669 $out .= $_."=".$testenv_vars->{$_}."\n";
678 return "" if ($envname eq "none");
679 return $target->getlog_env($running_envs{$envname});
685 return 1 if ($envname eq "none");
686 return $target->check_env($running_envs{$envname});
692 return if ($envname eq "none");
693 $target->teardown_env($running_envs{$envname});
694 delete $running_envs{$envname};
698 if ($opt_format eq "buildfarm") {
699 require output::buildfarm;
700 $msg_ops = new output::buildfarm();
701 } elsif ($opt_format eq "plain") {
702 require output::plain;
703 $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
704 } elsif ($opt_format eq "html") {
705 require output::html;
706 mkdir "test-results";
707 $msg_ops = new output::html("test-results", $statistics);
709 die("Invalid output format '$opt_format'");
712 if ($opt_no_lazy_setup) {
713 setup_env($_) foreach (keys %required_envs);
717 my $testenv_name = $ENV{SELFTEST_TESTENV};
718 $testenv_name = "dc" unless defined($testenv_name);
720 my $testenv_vars = setup_env($testenv_name);
722 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
724 my $envvarstr = exported_envvars_str($testenv_vars);
726 my $term = ($ENV{TERM} or "xterm");
727 system("$term -e 'echo -e \"
728 Welcome to the Samba4 Test environment '$testenv_name'
730 This matches the client environment used in make test
731 smbd is pid `cat \$PIDDIR/smbd.pid`
733 Some useful environment variables:
734 TORTURE_OPTIONS=\$TORTURE_OPTIONS
735 CONFIGURATION=\$CONFIGURATION
739 teardown_env($testenv_name);
744 $cmd =~ s/([\(\)])/\\$1/g;
746 my $envname = $$_[1];
749 $msg_ops->skip_testsuite($name);
750 $statistics->{SUITES_SKIPPED}++;
754 my $envvars = setup_env($envname);
755 if (not defined($envvars)) {
756 $statistics->{SUITES_FAIL}++;
757 $statistics->{TESTS_ERROR}++;
758 $msg_ops->missing_env($name, $envname);
762 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
764 if (defined($opt_analyse_cmd)) {
765 system("$opt_analyse_cmd \"$name\"");
768 teardown_env($envname) if ($opt_resetup_env);
774 teardown_env($_) foreach (keys %running_envs);
778 $statistics->{END_TIME} = time();
779 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
780 my $numfailed = $statistics->{SUITES_FAIL};
781 if ($numfailed == 0) {
782 my $ok = $statistics->{TESTS_EXPECTED_OK} +
783 $statistics->{TESTS_EXPECTED_FAIL};
784 print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
788 print "DURATION: $duration seconds\n";
792 # if there were any valgrind failures, show them
793 foreach (<$prefix/valgrind.log*>) {
795 system("grep DWARF2.CFI.reader $_ > /dev/null");
797 print "VALGRIND FAILURE\n";
803 if ($opt_format eq "buildfarm") {
804 print "TEST STATUS: $numfailed\n";