2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win|kvm>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME.TEST-NAME
72 The reason for a test can also be specified, by adding a hash sign (#) and the reason
77 Specify a file containing a list of tests that should be skipped. Possible
78 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
79 for the --expected-failures flag.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
136 my $opt_target = "samba4";
138 my $opt_socket_wrapper = 0;
139 my $opt_socket_wrapper_pcap = undef;
140 my $opt_socket_wrapper_keep_pcap = undef;
142 my $opt_immediate = 0;
143 my $opt_expected_failures = undef;
144 my @opt_exclude = ();
145 my @opt_include = ();
147 my $opt_image = undef;
150 my $opt_analyse_cmd = undef;
151 my $opt_resetup_env = undef;
152 my $opt_bindir = undef;
153 my $opt_no_lazy_setup = undef;
154 my $opt_format = "plain";
161 my @expected_failures = ();
168 TESTS_UNEXPECTED_OK => 0,
169 TESTS_EXPECTED_OK => 0,
170 TESTS_UNEXPECTED_FAIL => 0,
171 TESTS_EXPECTED_FAIL => 0,
178 my ($list, $fullname) = @_;
181 if ($fullname =~ /$$_[0]/) {
182 return ($$_[1]) if ($$_[1]);
183 return "NO REASON SPECIFIED";
190 sub expecting_failure($)
193 return find_in_list(\@expected_failures, $name);
200 return find_in_list(\@excludes, $name);
207 my ($state, $name) = @_;
209 return unless ($opt_socket_wrapper_pcap);
210 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
213 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
215 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
217 SocketWrapper::setup_pcap($pcap_file);
222 sub cleanup_pcap($$$)
224 my ($pcap_file, $expected_ret, $ret) = @_;
226 return unless ($opt_socket_wrapper_pcap);
227 return if ($opt_socket_wrapper_keep_pcap);
228 return unless ($expected_ret == $ret);
229 return unless defined($pcap_file);
234 sub run_testsuite($$$$$$)
236 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
237 my $pcap_file = setup_pcap($name);
239 $msg_ops->start_test([], $name);
241 open(RESULT, "$cmd 2>&1|");
242 my $expected_ret = parse_results(
243 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
245 my $envlog = getlog_env($envname);
246 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
248 $msg_ops->output_msg("CMD: $cmd\n");
250 my $ret = close(RESULT);
251 $ret = 0 unless $ret == 1;
254 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
256 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret,
260 cleanup_pcap($pcap_file, $expected_ret, $ret);
262 if (not $opt_socket_wrapper_keep_pcap and
263 defined($pcap_file)) {
264 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
267 if ($ret != $expected_ret) {
268 $statistics->{SUITES_FAIL}++;
269 exit(1) if ($opt_one);
272 return ($ret == $expected_ret);
277 print "Samba test runner
278 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
280 Usage: $Script [OPTIONS] PREFIX
283 --help this help page
284 --target=samba[34]|win|kvm Samba version to target
285 --testlist=FILE file to read available tests from
288 --prefix=DIR prefix to run tests in [st]
289 --srcdir=DIR source directory [.]
290 --builddir=DIR output directory [.]
293 --socket-wrapper-pcap save traffic to pcap directories
294 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
296 --socket-wrapper enable socket wrapper
297 --expected-failures=FILE specify list of tests that is guaranteed to fail
300 --ldap=openldap|fedora-ds back smbd onto specified ldap server
303 --bindir=PATH path to binaries
306 --image=PATH path to KVM image
309 --quick run quick overall test
310 --one abort when the first test fails
311 --immediate print test output for failed tests during run
313 --analyse-cmd CMD command to run after each test
318 my $result = GetOptions (
319 'help|h|?' => \$opt_help,
320 'target=s' => \$opt_target,
321 'prefix=s' => \$prefix,
322 'socket-wrapper' => \$opt_socket_wrapper,
323 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
324 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
325 'quick' => \$opt_quick,
327 'immediate' => \$opt_immediate,
328 'expected-failures=s' => \$opt_expected_failures,
329 'exclude=s' => \@opt_exclude,
330 'include=s' => \@opt_include,
331 'srcdir=s' => \$srcdir,
332 'builddir=s' => \$builddir,
333 'verbose' => \$opt_verbose,
334 'testenv' => \$opt_testenv,
336 'analyse-cmd=s' => \$opt_analyse_cmd,
337 'no-lazy-setup' => \$opt_no_lazy_setup,
338 'resetup-environment' => \$opt_resetup_env,
339 'bindir:s' => \$opt_bindir,
340 'format=s' => \$opt_format,
341 'image=s' => \$opt_image,
342 'testlist=s' => \@testlists
345 exit(1) if (not $result);
347 ShowHelp() if ($opt_help);
351 # quick hack to disable rpc validation when using valgrind - its way too slow
352 unless (defined($ENV{VALGRIND})) {
353 $ENV{VALIDATE} = "validate";
354 $ENV{MALLOC_CHECK_} = 2;
357 my $old_pwd = "$RealBin/..";
359 # Backwards compatibility:
360 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
361 if (defined($ENV{FEDORA_DS_ROOT})) {
368 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
371 $torture_maxtime *= 2;
378 die("using an empty prefix isn't allowed") unless $prefix ne "";
380 #Ensure we have the test prefix around
381 mkdir($prefix, 0777) unless -d $prefix;
383 my $prefix_abs = abs_path($prefix);
384 my $srcdir_abs = abs_path($srcdir);
386 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
387 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
389 $ENV{PREFIX} = $prefix;
390 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
391 $ENV{PREFIX_ABS} = $prefix_abs;
392 $ENV{SRCDIR} = $srcdir;
393 $ENV{SRCDIR_ABS} = $srcdir_abs;
395 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
396 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
397 $opt_format = "buildfarm";
400 my $tls_enabled = not $opt_quick;
401 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
402 $ENV{LDB_MODULES_PATH} = "$old_pwd/bin/modules/ldb";
403 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
404 if (defined($ENV{PKG_CONFIG_PATH})) {
405 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
407 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
409 # Required for smbscript:
410 $ENV{PATH} = "$old_pwd/bin:$old_pwd:$ENV{PATH}";
412 if ($opt_socket_wrapper_keep_pcap) {
413 # Socket wrapper keep pcap implies socket wrapper pcap
414 $opt_socket_wrapper_pcap = 1;
417 if ($opt_socket_wrapper_pcap) {
418 # Socket wrapper pcap implies socket wrapper
419 $opt_socket_wrapper = 1;
422 my $socket_wrapper_dir;
423 if ($opt_socket_wrapper) {
424 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
425 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
427 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
431 my $testenv_default = "none";
433 if ($opt_target eq "samba4") {
434 $testenv_default = "member";
435 require target::Samba4;
436 $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
437 } elsif ($opt_target eq "samba3") {
438 if ($opt_socket_wrapper and `$opt_bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
439 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
441 $testenv_default = "dc";
442 require target::Samba3;
443 $target = new Samba3($opt_bindir);
444 } elsif ($opt_target eq "win") {
445 die("Windows tests will not run with socket wrapper enabled.")
446 if ($opt_socket_wrapper);
447 $testenv_default = "dc";
448 require target::Windows;
449 $target = new Windows();
450 } elsif ($opt_target eq "kvm") {
451 die("Kvm tests will not run with socket wrapper enabled.")
452 if ($opt_socket_wrapper);
454 die("No image specified") unless ($opt_image);
455 $target = new Kvm($opt_image, undef);
459 # Start a Virtual Distributed Ethernet Switch
460 # Returns the pid of the switch.
462 sub start_vde_switch($)
466 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
468 open(PID, "$path/vde.pid");
476 # Stop a Virtual Distributed Ethernet Switch
477 sub stop_vde_switch($)
483 sub read_test_regexes($)
487 open(LF, "<$name") or die("unable to read $name: $!");
490 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
491 push (@ret, [$1, $4]);
493 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
494 push (@ret, [$_, undef]);
501 if (defined($opt_expected_failures)) {
502 @expected_failures = read_test_regexes($opt_expected_failures);
505 foreach (@opt_exclude) {
506 push (@excludes, read_test_regexes($_));
510 push (@includes, read_test_regexes("samba4-quick"));
513 foreach (@opt_include) {
514 push (@includes, read_test_regexes($_));
517 my $interfaces = join(',', ("127.0.0.6/8",
524 my $conffile = "$prefix_abs/client/client.conf";
526 sub write_clientconf($$)
528 my ($conffile, $vars) = @_;
530 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
532 if ( -d "$prefix/client/private" ) {
533 unlink <$prefix/client/private/*>;
535 mkdir("$prefix/client/private", 0777);
538 open(CF, ">$conffile");
539 print CF "[global]\n";
540 if (defined($ENV{VALGRIND})) {
541 print CF "\ticonv:native = true\n";
543 print CF "\ticonv:native = false\n";
545 print CF "\tnetbios name = client\n";
546 if (defined($vars->{DOMAIN})) {
547 print CF "\tworkgroup = $vars->{DOMAIN}\n";
549 if (defined($vars->{REALM})) {
550 print CF "\trealm = $vars->{REALM}\n";
552 if (defined($vars->{NCALRPCDIR})) {
553 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
555 if (defined($vars->{PIDDIR})) {
556 print CF "\tpid directory = $vars->{PIDDIR}\n";
558 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
559 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
561 if ($opt_socket_wrapper) {
562 print CF "\tinterfaces = $interfaces\n";
565 private dir = $prefix_abs/client/private
566 js include = $srcdir_abs/scripting/libjs
567 name resolve order = bcast
568 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
570 notify:inotify = false
572 system:anonymous = true
573 torture:basedir = $prefix_abs/client
574 #We don't want to pass our self-tests if the PAC code is wrong
575 gensec:require_pac = true
576 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
581 my @torture_options = ();
582 push (@torture_options, "--configfile=$conffile");
583 # ensure any one smbtorture call doesn't run too long
584 push (@torture_options, "--maximum-runtime=$torture_maxtime");
585 push (@torture_options, "--target=$opt_target");
586 push (@torture_options, "--basedir=$prefix_abs");
587 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
588 push (@torture_options, "--format=subunit");
589 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
591 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
592 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
596 my $testsdir = "$srcdir/selftest";
597 $ENV{SMB_CONF_PATH} = "$conffile";
598 $ENV{CONFIGURATION} = "--configfile=$conffile";
600 my %required_envs = ();
607 open(IN, $filename) or die("Unable to open $filename: $!");
610 if ($_ eq "-- TEST --\n") {
617 if (not defined($tests) or $name =~ /$tests/) {
618 $required_envs{$env} = 1;
619 push (@ret, [$name, $env, $cmdline]);
625 close(IN) or die("Error creating recipe");
629 if ($#testlists == -1) {
630 die("No testlists specified");
634 foreach my $fn (@testlists) {
635 foreach (read_testlist($fn)) {
637 next if (@includes and not find_in_list(\@includes, $name));
638 push (@available, $_);
643 if ($opt_format eq "buildfarm") {
644 require output::buildfarm;
645 $msg_ops = new output::buildfarm($statistics);
646 } elsif ($opt_format eq "plain") {
647 require output::plain;
648 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
649 } elsif ($opt_format eq "html") {
650 require output::html;
651 mkdir("test-results", 0777);
652 $msg_ops = new output::html("test-results", $statistics);
654 die("Invalid output format '$opt_format'");
658 foreach (@available) {
660 my $skipreason = skip($name);
662 $msg_ops->skip_testsuite($name, $skipreason);
669 print STDERR "No tests to run\n";
673 my $suitestotal = $#todo + 1;
677 my %running_envs = ();
679 my @exported_envvars = (
684 # domain controller stuff
704 "WINBINDD_SOCKET_DIR",
705 "WINBINDD_PRIV_PIPE_DIR"
708 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
710 teardown_env($_) foreach(keys %running_envs);
711 die("Received signal $signame");
719 if ($envname eq "none") {
721 } elsif (defined($running_envs{$envname})) {
722 $testenv_vars = $running_envs{$envname};
723 if (not $target->check_env($testenv_vars)) {
724 $testenv_vars = undef;
727 $testenv_vars = $target->setup_env($envname, $prefix);
730 return undef unless defined($testenv_vars);
732 $running_envs{$envname} = $testenv_vars;
734 SocketWrapper::set_default_iface(6);
735 write_clientconf($conffile, $testenv_vars);
737 foreach (@exported_envvars) {
738 if (defined($testenv_vars->{$_})) {
739 $ENV{$_} = $testenv_vars->{$_};
745 return $testenv_vars;
748 sub exported_envvars_str($)
750 my ($testenv_vars) = @_;
753 foreach (@exported_envvars) {
754 next unless defined($testenv_vars->{$_});
755 $out .= $_."=".$testenv_vars->{$_}."\n";
764 return "" if ($envname eq "none");
765 return $target->getlog_env($running_envs{$envname});
771 return 1 if ($envname eq "none");
772 return $target->check_env($running_envs{$envname});
778 return if ($envname eq "none");
779 $target->teardown_env($running_envs{$envname});
780 delete $running_envs{$envname};
783 if ($opt_no_lazy_setup) {
784 setup_env($_) foreach (keys %required_envs);
788 my $testenv_name = $ENV{SELFTEST_TESTENV};
789 $testenv_name = $testenv_default unless defined($testenv_name);
791 my $testenv_vars = setup_env($testenv_name);
793 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
795 my $envvarstr = exported_envvars_str($testenv_vars);
797 my $term = ($ENV{TERM} or "xterm");
798 system("$term -e 'echo -e \"
799 Welcome to the Samba4 Test environment '$testenv_name'
801 This matches the client environment used in make test
802 smbd is pid `cat \$PIDDIR/smbd.pid`
804 Some useful environment variables:
805 TORTURE_OPTIONS=\$TORTURE_OPTIONS
806 CONFIGURATION=\$CONFIGURATION
809 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
810 teardown_env($testenv_name);
815 $cmd =~ s/([\(\)])/\\$1/g;
817 my $envname = $$_[1];
819 my $envvars = setup_env($envname);
820 if (not defined($envvars)) {
821 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
825 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
828 if (defined($opt_analyse_cmd)) {
829 system("$opt_analyse_cmd \"$name\"");
832 teardown_env($envname) if ($opt_resetup_env);
838 teardown_env($_) foreach (keys %running_envs);
846 # if there were any valgrind failures, show them
847 foreach (<$prefix/valgrind.log*>) {
849 system("grep DWARF2.CFI.reader $_ > /dev/null");
851 print "VALGRIND FAILURE\n";
857 if ($opt_format eq "buildfarm") {
858 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
861 exit $statistics->{SUITES_FAIL};