2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME/TEST-NAME
72 The reason for a test can also be specified, by adding a hash sign (#) and the reason
77 Specify a file containing a list of tests that should be skipped. Possible candidates are
78 tests that segfault the server, flip or don't end. The format of this file is the same as
79 for the --expected-failures flag.
83 Abort as soon as one test fails.
87 Load a list of tests from the specified location.
95 =item I<SMBD_VALGRIND>
97 =item I<TORTURE_MAXTIME>
109 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
119 use FindBin qw($RealBin $Script);
123 use Cwd qw(abs_path);
125 use Subunit qw(parse_results);
132 my $opt_target = "samba4";
134 my $opt_socket_wrapper = 0;
135 my $opt_socket_wrapper_pcap = undef;
136 my $opt_socket_wrapper_keep_pcap = undef;
138 my $opt_immediate = 0;
139 my $opt_expected_failures = undef;
144 my $opt_analyse_cmd = undef;
145 my $opt_resetup_env = undef;
146 my $opt_bindir = undef;
147 my $opt_no_lazy_setup = undef;
148 my $opt_format = "plain";
149 my @opt_testlists = ();
155 my @expected_failures = ();
159 START_TIME => time(),
165 TESTS_UNEXPECTED_OK => 0,
166 TESTS_EXPECTED_OK => 0,
167 TESTS_UNEXPECTED_FAIL => 0,
168 TESTS_EXPECTED_FAIL => 0,
175 my ($list, $fullname) = @_;
178 if ($fullname =~ /$$_[0]/) {
179 return ($$_[1]) if ($$_[1]);
180 return "NO REASON SPECIFIED";
187 sub expecting_failure($)
190 return find_in_list(\@expected_failures, $name);
196 return find_in_list(\@skips, $name);
205 return unless ($opt_socket_wrapper_pcap);
206 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
208 my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
209 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
211 $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
213 SocketWrapper::setup_pcap($state->{PCAP_FILE});
216 sub cleanup_pcap($$$)
218 my ($state, $expected_ret, $ret) = @_;
220 return unless ($opt_socket_wrapper_pcap);
221 return if ($opt_socket_wrapper_keep_pcap);
222 return unless ($expected_ret == $ret);
223 return unless defined($state->{PCAP_FILE});
225 unlink($state->{PCAP_FILE});
226 $state->{PCAP_FILE} = undef;
229 sub run_testsuite($$$$$$$)
231 my ($envname, $envvars, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
238 TOTAL => $totalsuites,
242 setup_pcap($msg_state);
244 open(RESULT, "$cmd 2>&1|");
245 $msg_ops->start_testsuite($msg_state);
247 my $expected_ret = parse_results(
248 $msg_ops, $msg_state, $statistics, *RESULT, \&expecting_failure);
250 my $ret = close(RESULT);
252 cleanup_pcap($msg_state, $expected_ret, $ret);
254 $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
255 getlog_env($msg_state->{ENVNAME}));
257 if (not $opt_socket_wrapper_keep_pcap and
258 defined($msg_state->{PCAP_FILE})) {
259 $msg_ops->output_msg($msg_state,
260 "PCAP FILE: $msg_state->{PCAP_FILE}\n");
263 if ($ret != $expected_ret) {
264 $statistics->{SUITES_FAIL}++;
265 exit(1) if ($opt_one);
267 $statistics->{SUITES_OK}++;
270 return ($ret == $expected_ret);
275 print "Samba test runner
276 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
278 Usage: $Script [OPTIONS] PREFIX
281 --help this help page
282 --target=samba4|samba3|win Samba version to target
283 --testlist=FILE file to read available tests from
286 --prefix=DIR prefix to run tests in [st]
287 --srcdir=DIR source directory [.]
288 --builddir=DIR output directory [.]
291 --socket-wrapper-pcap=DIR save traffic to pcap directories
292 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
294 --socket-wrapper enable socket wrapper
295 --expected-failures=FILE specify list of tests that is guaranteed to fail
298 --ldap=openldap|fedora-ds back smbd onto specified ldap server
301 --bindir=PATH path to binaries
304 --quick run quick overall test
305 --one abort when the first test fails
306 --immediate print test output for failed tests during run
308 --analyse-cmd CMD command to run after each test
313 my $result = GetOptions (
314 'help|h|?' => \$opt_help,
315 'target=s' => \$opt_target,
316 'prefix=s' => \$prefix,
317 'socket-wrapper' => \$opt_socket_wrapper,
318 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
319 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
320 'quick' => \$opt_quick,
322 'immediate' => \$opt_immediate,
323 'expected-failures=s' => \$opt_expected_failures,
324 'skip=s' => \@opt_skip,
325 'srcdir=s' => \$srcdir,
326 'builddir=s' => \$builddir,
327 'verbose' => \$opt_verbose,
328 'testenv' => \$opt_testenv,
330 'analyse-cmd=s' => \$opt_analyse_cmd,
331 'no-lazy-setup' => \$opt_no_lazy_setup,
332 'resetup-environment' => \$opt_resetup_env,
333 'bindir:s' => \$opt_bindir,
334 'format=s' => \$opt_format,
335 'testlist=s' => \@opt_testlists
338 exit(1) if (not $result);
340 ShowHelp() if ($opt_help);
344 # quick hack to disable rpc validation when using valgrind - its way too slow
345 unless (defined($ENV{VALGRIND})) {
346 $ENV{VALIDATE} = "validate";
347 $ENV{MALLOC_CHECK_} = 2;
350 my $old_pwd = "$RealBin/..";
352 # Backwards compatibility:
353 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
354 if (defined($ENV{FEDORA_DS_PREFIX})) {
361 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
364 $torture_maxtime *= 2;
371 die("using an empty prefix isn't allowed") unless $prefix ne "";
373 #Ensure we have the test prefix around
374 mkdir($prefix, 0777) unless -d $prefix;
376 my $prefix_abs = abs_path($prefix);
377 my $srcdir_abs = abs_path($srcdir);
379 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
380 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
382 $ENV{PREFIX} = $prefix;
383 $ENV{PREFIX_ABS} = $prefix_abs;
384 $ENV{SRCDIR} = $srcdir;
385 $ENV{SRCDIR_ABS} = $srcdir_abs;
387 my $tls_enabled = not $opt_quick;
388 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
389 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
390 $opt_format = "buildfarm";
393 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
394 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
395 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
396 if (defined($ENV{LD_LIBRARY_PATH})) {
397 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
399 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
401 if (defined($ENV{PKG_CONFIG_PATH})) {
402 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
404 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
406 # Required for smbscript:
407 $ENV{PATH} = "$old_pwd/bin:$old_pwd:$ENV{PATH}";
410 if ($opt_socket_wrapper_pcap) {
411 # Socket wrapper pcap implies socket wrapper
412 $opt_socket_wrapper = 1;
415 my $socket_wrapper_dir;
416 if ($opt_socket_wrapper) {
417 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
418 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
420 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
425 if ($opt_target eq "samba4") {
426 $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
427 } elsif ($opt_target eq "samba3") {
428 if ($opt_socket_wrapper and `$opt_bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
429 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
432 $target = new Samba3($opt_bindir);
433 } elsif ($opt_target eq "win") {
434 die("Windows tests will not run with socket wrapper enabled.")
435 if ($opt_socket_wrapper);
436 $target = new Windows();
439 sub read_test_regexes($)
443 open(LF, "<$name") or die("unable to read $name: $!");
446 if (/^(.*?)([ \t]+)\#(.*)$/) {
447 push (@ret, [$1, $3]);
449 s/^(.*?)([ \t]+)\#(.*)$//;
450 push (@ret, [$_, undef]);
457 if (defined($opt_expected_failures)) {
458 @expected_failures = read_test_regexes($opt_expected_failures);
461 foreach (@opt_skip) {
462 push (@skips, read_test_regexes($_));
465 my $interfaces = join(',', ("127.0.0.6/8",
472 my $conffile = "$prefix_abs/client/client.conf";
474 sub write_clientconf($$)
476 my ($conffile, $vars) = @_;
478 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
480 if ( -d "$prefix/client/private" ) {
481 unlink <$prefix/client/private/*>;
483 mkdir("$prefix/client/private", 0777);
486 open(CF, ">$conffile");
487 print CF "[global]\n";
488 if (defined($ENV{VALGRIND})) {
489 print CF "\ticonv:native = true\n";
491 print CF "\ticonv:native = false\n";
493 print CF "\tnetbios name = client\n";
494 if (defined($vars->{DOMAIN})) {
495 print CF "\tworkgroup = $vars->{DOMAIN}\n";
497 if (defined($vars->{REALM})) {
498 print CF "\trealm = $vars->{REALM}\n";
500 if (defined($vars->{NCALRPCDIR})) {
501 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
503 if (defined($vars->{PIDDIR})) {
504 print CF "\tpid directory = $vars->{PIDDIR}\n";
506 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
507 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
510 private dir = $prefix_abs/client/private
511 js include = $srcdir_abs/scripting/libjs
512 name resolve order = bcast
513 interfaces = $interfaces
514 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
516 notify:inotify = false
518 system:anonymous = true
519 torture:basedir = $prefix_abs/client
520 #We don't want to pass our self-tests if the PAC code is wrong
521 gensec:require_pac = true
527 my @torture_options = ();
528 push (@torture_options, "--configfile=$conffile");
529 # ensure any one smbtorture call doesn't run too long
530 push (@torture_options, "--maximum-runtime=$torture_maxtime");
531 push (@torture_options, "--target=$opt_target");
532 push (@torture_options, "--basedir=$prefix_abs");
533 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
534 push (@torture_options, "--format=subunit");
535 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
537 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
538 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
542 my $testsdir = "$srcdir/selftest";
543 $ENV{SMB_CONF_PATH} = "$conffile";
544 $ENV{CONFIGURATION} = "--configfile=$conffile";
546 my %required_envs = ();
553 open(IN, $filename) or die("Unable to open $filename: $!");
556 if ($_ eq "-- TEST --\n") {
563 if (not defined($tests) or $name =~ /$tests/) {
564 $required_envs{$env} = 1;
565 push (@ret, [$name, $env, $cmdline]);
571 close(IN) or die("Error creating recipe");
576 @todo = read_testlist("$testsdir/tests_quick.sh|");
578 @todo = read_testlist("$testsdir/tests_all.sh|");
581 foreach (@opt_testlists) {
582 push(@todo, read_testlist($_));
585 my $suitestotal = $#todo + 1;
589 my %running_envs = ();
591 my @exported_envvars = (
596 # domain controller stuff
616 "WINBINDD_SOCKET_DIR"
619 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
621 teardown_env($_) foreach(keys %running_envs);
622 die("Received signal $signame");
630 if ($envname eq "none") {
632 } elsif (defined($running_envs{$envname})) {
633 $testenv_vars = $running_envs{$envname};
634 if (not $target->check_env($testenv_vars)) {
635 $testenv_vars = undef;
638 $testenv_vars = $target->setup_env($envname, $prefix);
641 return undef unless defined($testenv_vars);
643 $running_envs{$envname} = $testenv_vars;
645 SocketWrapper::set_default_iface(6);
646 write_clientconf($conffile, $testenv_vars);
648 foreach (@exported_envvars) {
649 if (defined($testenv_vars->{$_})) {
650 $ENV{$_} = $testenv_vars->{$_};
656 return $testenv_vars;
659 sub exported_envvars_str($)
661 my ($testenv_vars) = @_;
664 foreach (@exported_envvars) {
665 next unless defined($testenv_vars->{$_});
666 $out .= $_."=".$testenv_vars->{$_}."\n";
675 return "" if ($envname eq "none");
676 return $target->getlog_env($running_envs{$envname});
682 return 1 if ($envname eq "none");
683 return $target->check_env($running_envs{$envname});
689 return if ($envname eq "none");
690 $target->teardown_env($running_envs{$envname});
691 delete $running_envs{$envname};
695 if ($opt_format eq "buildfarm") {
696 require output::buildfarm;
697 $msg_ops = new output::buildfarm();
698 } elsif ($opt_format eq "plain") {
699 require output::plain;
700 $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
701 } elsif ($opt_format eq "html") {
702 require output::html;
703 mkdir("test-results", 0777);
704 $msg_ops = new output::html("test-results", $statistics);
706 die("Invalid output format '$opt_format'");
709 if ($opt_no_lazy_setup) {
710 setup_env($_) foreach (keys %required_envs);
714 my $testenv_name = $ENV{SELFTEST_TESTENV};
715 $testenv_name = "member" unless defined($testenv_name);
717 my $testenv_vars = setup_env($testenv_name);
719 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
721 my $envvarstr = exported_envvars_str($testenv_vars);
723 my $term = ($ENV{TERM} or "xterm");
724 system("$term -e 'echo -e \"
725 Welcome to the Samba4 Test environment '$testenv_name'
727 This matches the client environment used in make test
728 smbd is pid `cat \$PIDDIR/smbd.pid`
730 Some useful environment variables:
731 TORTURE_OPTIONS=\$TORTURE_OPTIONS
732 CONFIGURATION=\$CONFIGURATION
736 teardown_env($testenv_name);
741 $cmd =~ s/([\(\)])/\\$1/g;
743 my $envname = $$_[1];
745 my $skipreason = skip($name);
747 $msg_ops->skip_testsuite($envname, $name, $skipreason);
748 $statistics->{SUITES_SKIPPED}++;
752 my $envvars = setup_env($envname);
753 if (not defined($envvars)) {
754 $statistics->{SUITES_SKIPPED}++;
755 $msg_ops->missing_env($name, $envname);
759 run_testsuite($envname, $envvars, $name, $cmd, $i, $suitestotal,
762 if (defined($opt_analyse_cmd)) {
763 system("$opt_analyse_cmd \"$name\"");
766 teardown_env($envname) if ($opt_resetup_env);
772 teardown_env($_) foreach (keys %running_envs);
776 $statistics->{END_TIME} = time();
777 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
779 print "DURATION: $duration seconds\n";
783 # if there were any valgrind failures, show them
784 foreach (<$prefix/valgrind.log*>) {
786 system("grep DWARF2.CFI.reader $_ > /dev/null");
788 print "VALGRIND FAILURE\n";
794 if ($opt_format eq "buildfarm") {
795 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
798 exit $statistics->{SUITES_FAIL};