2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME/TEST-NAME
72 The reason for a test can also be specified, by adding a hash sign (#) and the reason
77 Specify a file containing a list of tests that should be skipped. Possible
78 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
79 for the --expected-failures flag.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
139 my $opt_target = "samba4";
141 my $opt_socket_wrapper = 0;
142 my $opt_socket_wrapper_pcap = undef;
143 my $opt_socket_wrapper_keep_pcap = undef;
145 my $opt_immediate = 0;
146 my $opt_expected_failures = undef;
147 my @opt_exclude = ();
148 my @opt_include = ();
152 my $opt_analyse_cmd = undef;
153 my $opt_resetup_env = undef;
154 my $opt_bindir = undef;
155 my $opt_no_lazy_setup = undef;
156 my $opt_format = "plain";
163 my @expected_failures = ();
168 START_TIME => time(),
174 TESTS_UNEXPECTED_OK => 0,
175 TESTS_EXPECTED_OK => 0,
176 TESTS_UNEXPECTED_FAIL => 0,
177 TESTS_EXPECTED_FAIL => 0,
184 my ($list, $fullname) = @_;
187 if ($fullname =~ /$$_[0]/) {
188 return ($$_[1]) if ($$_[1]);
189 return "NO REASON SPECIFIED";
196 sub expecting_failure($)
199 return find_in_list(\@expected_failures, $name);
206 return find_in_list(\@excludes, $name);
215 return unless ($opt_socket_wrapper_pcap);
216 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
218 my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
219 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
221 $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
223 SocketWrapper::setup_pcap($state->{PCAP_FILE});
226 sub cleanup_pcap($$$)
228 my ($state, $expected_ret, $ret) = @_;
230 return unless ($opt_socket_wrapper_pcap);
231 return if ($opt_socket_wrapper_keep_pcap);
232 return unless ($expected_ret == $ret);
233 return unless defined($state->{PCAP_FILE});
235 unlink($state->{PCAP_FILE});
236 $state->{PCAP_FILE} = undef;
239 sub run_testsuite($$$$$$$)
241 my ($envname, $envvars, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
248 TOTAL => $totalsuites,
252 setup_pcap($msg_state);
254 open(RESULT, "$cmd 2>&1|");
255 $msg_ops->start_testsuite($name, $msg_state);
257 my $expected_ret = parse_results(
258 $msg_ops, $msg_state, $statistics, *RESULT, \&expecting_failure);
260 my $ret = close(RESULT);
261 $ret = 0 unless $ret == 1;
263 cleanup_pcap($msg_state, $expected_ret, $ret);
265 $msg_ops->end_testsuite($name, $msg_state, $expected_ret, $ret,
266 getlog_env($msg_state->{ENVNAME}));
268 if (not $opt_socket_wrapper_keep_pcap and
269 defined($msg_state->{PCAP_FILE})) {
270 $msg_ops->output_msg($msg_state,
271 "PCAP FILE: $msg_state->{PCAP_FILE}\n");
274 if ($ret != $expected_ret) {
275 $statistics->{SUITES_FAIL}++;
276 exit(1) if ($opt_one);
278 $statistics->{SUITES_OK}++;
281 return ($ret == $expected_ret);
286 print "Samba test runner
287 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
289 Usage: $Script [OPTIONS] PREFIX
292 --help this help page
293 --target=samba4|samba3|win Samba version to target
294 --testlist=FILE file to read available tests from
297 --prefix=DIR prefix to run tests in [st]
298 --srcdir=DIR source directory [.]
299 --builddir=DIR output directory [.]
302 --socket-wrapper-pcap save traffic to pcap directories
303 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
305 --socket-wrapper enable socket wrapper
306 --expected-failures=FILE specify list of tests that is guaranteed to fail
309 --ldap=openldap|fedora-ds back smbd onto specified ldap server
312 --bindir=PATH path to binaries
315 --quick run quick overall test
316 --one abort when the first test fails
317 --immediate print test output for failed tests during run
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'immediate' => \$opt_immediate,
334 'expected-failures=s' => \$opt_expected_failures,
335 'exclude=s' => \@opt_exclude,
336 'include=s' => \@opt_include,
337 'srcdir=s' => \$srcdir,
338 'builddir=s' => \$builddir,
339 'verbose' => \$opt_verbose,
340 'testenv' => \$opt_testenv,
342 'analyse-cmd=s' => \$opt_analyse_cmd,
343 'no-lazy-setup' => \$opt_no_lazy_setup,
344 'resetup-environment' => \$opt_resetup_env,
345 'bindir:s' => \$opt_bindir,
346 'format=s' => \$opt_format,
347 'testlist=s' => \@testlists
350 exit(1) if (not $result);
352 ShowHelp() if ($opt_help);
356 # quick hack to disable rpc validation when using valgrind - its way too slow
357 unless (defined($ENV{VALGRIND})) {
358 $ENV{VALIDATE} = "validate";
359 $ENV{MALLOC_CHECK_} = 2;
362 my $old_pwd = "$RealBin/..";
364 # Backwards compatibility:
365 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
366 if (defined($ENV{FEDORA_DS_PREFIX})) {
373 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
376 $torture_maxtime *= 2;
383 die("using an empty prefix isn't allowed") unless $prefix ne "";
385 #Ensure we have the test prefix around
386 mkdir($prefix, 0777) unless -d $prefix;
388 my $prefix_abs = abs_path($prefix);
389 my $srcdir_abs = abs_path($srcdir);
391 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
392 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
394 $ENV{PREFIX} = $prefix;
395 $ENV{PREFIX_ABS} = $prefix_abs;
396 $ENV{SRCDIR} = $srcdir;
397 $ENV{SRCDIR_ABS} = $srcdir_abs;
399 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
400 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
401 $opt_format = "buildfarm";
404 my $tls_enabled = not $opt_quick;
405 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
406 $ENV{LDB_MODULES_PATH} = "$old_pwd/bin/modules/ldb";
407 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
408 if (defined($ENV{PKG_CONFIG_PATH})) {
409 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
411 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
413 # Required for smbscript:
414 $ENV{PATH} = "$old_pwd/bin:$old_pwd:$ENV{PATH}";
416 if ($opt_socket_wrapper_keep_pcap) {
417 # Socket wrapper keep pcap implies socket wrapper pcap
418 $opt_socket_wrapper_pcap = 1;
421 if ($opt_socket_wrapper_pcap) {
422 # Socket wrapper pcap implies socket wrapper
423 $opt_socket_wrapper = 1;
426 my $socket_wrapper_dir;
427 if ($opt_socket_wrapper) {
428 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
429 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
431 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
435 my $testenv_default = "none";
437 if ($opt_target eq "samba4") {
438 $testenv_default = "member";
439 $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
440 } elsif ($opt_target eq "samba3") {
441 if ($opt_socket_wrapper and `$opt_bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
442 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
444 $testenv_default = "dc";
445 $target = new Samba3($opt_bindir);
446 } elsif ($opt_target eq "win") {
447 die("Windows tests will not run with socket wrapper enabled.")
448 if ($opt_socket_wrapper);
449 $testenv_default = "dc";
450 $target = new Windows();
453 sub read_test_regexes($)
457 open(LF, "<$name") or die("unable to read $name: $!");
460 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
461 push (@ret, [$1, $4]);
463 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
464 push (@ret, [$_, undef]);
471 if (defined($opt_expected_failures)) {
472 @expected_failures = read_test_regexes($opt_expected_failures);
475 foreach (@opt_exclude) {
476 push (@excludes, read_test_regexes($_));
480 push (@includes, read_test_regexes("samba4-quick"));
483 foreach (@opt_include) {
484 push (@includes, read_test_regexes($_));
487 my $interfaces = join(',', ("127.0.0.6/8",
494 my $conffile = "$prefix_abs/client/client.conf";
496 sub write_clientconf($$)
498 my ($conffile, $vars) = @_;
500 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
502 if ( -d "$prefix/client/private" ) {
503 unlink <$prefix/client/private/*>;
505 mkdir("$prefix/client/private", 0777);
508 open(CF, ">$conffile");
509 print CF "[global]\n";
510 if (defined($ENV{VALGRIND})) {
511 print CF "\ticonv:native = true\n";
513 print CF "\ticonv:native = false\n";
515 print CF "\tnetbios name = client\n";
516 if (defined($vars->{DOMAIN})) {
517 print CF "\tworkgroup = $vars->{DOMAIN}\n";
519 if (defined($vars->{REALM})) {
520 print CF "\trealm = $vars->{REALM}\n";
522 if (defined($vars->{NCALRPCDIR})) {
523 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
525 if (defined($vars->{PIDDIR})) {
526 print CF "\tpid directory = $vars->{PIDDIR}\n";
528 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
529 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
532 private dir = $prefix_abs/client/private
533 js include = $srcdir_abs/scripting/libjs
534 name resolve order = bcast
535 interfaces = $interfaces
536 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
538 notify:inotify = false
540 system:anonymous = true
541 torture:basedir = $prefix_abs/client
542 #We don't want to pass our self-tests if the PAC code is wrong
543 gensec:require_pac = true
544 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
550 my @torture_options = ();
551 push (@torture_options, "--configfile=$conffile");
552 # ensure any one smbtorture call doesn't run too long
553 push (@torture_options, "--maximum-runtime=$torture_maxtime");
554 push (@torture_options, "--target=$opt_target");
555 push (@torture_options, "--basedir=$prefix_abs");
556 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
557 push (@torture_options, "--format=subunit");
558 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
560 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
561 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
565 my $testsdir = "$srcdir/selftest";
566 $ENV{SMB_CONF_PATH} = "$conffile";
567 $ENV{CONFIGURATION} = "--configfile=$conffile";
569 my %required_envs = ();
576 open(IN, $filename) or die("Unable to open $filename: $!");
579 if ($_ eq "-- TEST --\n") {
586 if (not defined($tests) or $name =~ /$tests/) {
587 $required_envs{$env} = 1;
588 push (@ret, [$name, $env, $cmdline]);
594 close(IN) or die("Error creating recipe");
598 if ($#testlists == -1) {
599 die("No testlists specified");
603 if ($opt_format eq "buildfarm") {
604 require output::buildfarm;
605 $msg_ops = new output::buildfarm($statistics);
606 } elsif ($opt_format eq "plain") {
607 require output::plain;
608 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics);
609 } elsif ($opt_format eq "html") {
610 require output::html;
611 mkdir("test-results", 0777);
612 $msg_ops = new output::html("test-results", $statistics);
614 die("Invalid output format '$opt_format'");
617 foreach my $fn (@testlists) {
618 foreach (read_testlist($fn)) {
620 next if (@includes and not find_in_list(\@includes, $name));
621 my $skipreason = skip($name);
623 $msg_ops->skip_testsuite($name, $skipreason);
624 $statistics->{SUITES_SKIPPED}++;
632 print STDERR "No tests to run\n";
636 my $suitestotal = $#todo + 1;
640 my %running_envs = ();
642 my @exported_envvars = (
647 # domain controller stuff
667 "WINBINDD_SOCKET_DIR",
668 "WINBINDD_PRIV_PIPE_DIR"
671 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
673 teardown_env($_) foreach(keys %running_envs);
674 die("Received signal $signame");
682 if ($envname eq "none") {
684 } elsif (defined($running_envs{$envname})) {
685 $testenv_vars = $running_envs{$envname};
686 if (not $target->check_env($testenv_vars)) {
687 $testenv_vars = undef;
690 $testenv_vars = $target->setup_env($envname, $prefix);
693 return undef unless defined($testenv_vars);
695 $running_envs{$envname} = $testenv_vars;
697 SocketWrapper::set_default_iface(6);
698 write_clientconf($conffile, $testenv_vars);
700 foreach (@exported_envvars) {
701 if (defined($testenv_vars->{$_})) {
702 $ENV{$_} = $testenv_vars->{$_};
708 return $testenv_vars;
711 sub exported_envvars_str($)
713 my ($testenv_vars) = @_;
716 foreach (@exported_envvars) {
717 next unless defined($testenv_vars->{$_});
718 $out .= $_."=".$testenv_vars->{$_}."\n";
727 return "" if ($envname eq "none");
728 return $target->getlog_env($running_envs{$envname});
734 return 1 if ($envname eq "none");
735 return $target->check_env($running_envs{$envname});
741 return if ($envname eq "none");
742 $target->teardown_env($running_envs{$envname});
743 delete $running_envs{$envname};
747 if ($opt_no_lazy_setup) {
748 setup_env($_) foreach (keys %required_envs);
752 my $testenv_name = $ENV{SELFTEST_TESTENV};
753 $testenv_name = $testenv_default unless defined($testenv_name);
755 my $testenv_vars = setup_env($testenv_name);
757 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
759 my $envvarstr = exported_envvars_str($testenv_vars);
761 my $term = ($ENV{TERM} or "xterm");
762 system("$term -e 'echo -e \"
763 Welcome to the Samba4 Test environment '$testenv_name'
765 This matches the client environment used in make test
766 smbd is pid `cat \$PIDDIR/smbd.pid`
768 Some useful environment variables:
769 TORTURE_OPTIONS=\$TORTURE_OPTIONS
770 CONFIGURATION=\$CONFIGURATION
774 teardown_env($testenv_name);
779 $cmd =~ s/([\(\)])/\\$1/g;
781 my $envname = $$_[1];
783 my $envvars = setup_env($envname);
784 if (not defined($envvars)) {
785 $statistics->{SUITES_SKIPPED}++;
786 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
790 run_testsuite($envname, $envvars, $name, $cmd, $i, $suitestotal,
793 if (defined($opt_analyse_cmd)) {
794 system("$opt_analyse_cmd \"$name\"");
797 teardown_env($envname) if ($opt_resetup_env);
803 teardown_env($_) foreach (keys %running_envs);
807 $statistics->{END_TIME} = time();
808 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
810 print "DURATION: $duration seconds\n";
814 # if there were any valgrind failures, show them
815 foreach (<$prefix/valgrind.log*>) {
817 system("grep DWARF2.CFI.reader $_ > /dev/null");
819 print "VALGRIND FAILURE\n";
825 if ($opt_format eq "buildfarm") {
826 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
829 exit $statistics->{SUITES_FAIL};