2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win|kvm>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME.TEST-NAME
72 The reason for a test can also be specified, by adding a hash sign (#) and the reason
77 Specify a file containing a list of tests that should be skipped. Possible
78 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
79 for the --expected-failures flag.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
136 my $opt_target = "samba4";
138 my $opt_socket_wrapper = 0;
139 my $opt_socket_wrapper_pcap = undef;
140 my $opt_socket_wrapper_keep_pcap = undef;
142 my $opt_immediate = 0;
143 my $opt_expected_failures = undef;
144 my @opt_exclude = ();
145 my @opt_include = ();
147 my $opt_image = undef;
150 my $opt_analyse_cmd = undef;
151 my $opt_resetup_env = undef;
152 my $opt_bindir = undef;
153 my $opt_no_lazy_setup = undef;
154 my $opt_format = "plain";
161 my @expected_failures = ();
168 TESTS_UNEXPECTED_OK => 0,
169 TESTS_EXPECTED_OK => 0,
170 TESTS_UNEXPECTED_FAIL => 0,
171 TESTS_EXPECTED_FAIL => 0,
178 my ($list, $fullname) = @_;
181 if ($fullname =~ /$$_[0]/) {
182 return ($$_[1]) if ($$_[1]);
183 return "NO REASON SPECIFIED";
190 sub expecting_failure($)
193 return find_in_list(\@expected_failures, $name);
200 return find_in_list(\@excludes, $name);
209 return unless ($opt_socket_wrapper_pcap);
210 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
213 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
215 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
217 SocketWrapper::setup_pcap($pcap_file);
222 sub cleanup_pcap($$$)
224 my ($pcap_file, $expected_ret, $ret) = @_;
226 return unless ($opt_socket_wrapper_pcap);
227 return if ($opt_socket_wrapper_keep_pcap);
228 return unless ($expected_ret == $ret);
229 return unless defined($pcap_file);
234 sub run_testsuite($$$$$$)
236 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
237 my $pcap_file = setup_pcap($name);
239 $msg_ops->start_test([], $name);
241 unless (open(RESULT, "$cmd 2>&1|")) {
242 $statistics->{TESTS_ERROR}++;
243 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
244 $statistics->{SUITES_FAIL}++;
248 my $expected_ret = parse_results(
249 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
251 my $envlog = getlog_env($envname);
252 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
254 $msg_ops->output_msg("CMD: $cmd\n");
256 my $ret = close(RESULT);
257 $ret = 0 unless $ret == 1;
259 my $exitcode = $? >> 8;
262 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
264 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
267 cleanup_pcap($pcap_file, $expected_ret, $ret);
269 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
270 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
273 if ($ret != $expected_ret) {
274 $statistics->{SUITES_FAIL}++;
275 exit(1) if ($opt_one);
278 return ($ret == $expected_ret);
283 print "Samba test runner
284 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
286 Usage: $Script [OPTIONS] TESTNAME-REGEX
289 --help this help page
290 --target=samba[34]|win|kvm Samba version to target
291 --testlist=FILE file to read available tests from
294 --prefix=DIR prefix to run tests in [st]
295 --srcdir=DIR source directory [.]
296 --builddir=DIR output directory [.]
299 --socket-wrapper-pcap save traffic to pcap directories
300 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
302 --socket-wrapper enable socket wrapper
303 --bindir=PATH path to target binaries
304 --expected-failures=FILE specify list of tests that is guaranteed to fail
307 --ldap=openldap|fedora-ds back samba onto specified ldap server
310 --image=PATH path to KVM image
313 --quick run quick overall test
314 --one abort when the first test fails
315 --immediate print test output for failed tests during run
317 --analyse-cmd CMD command to run after each test
322 my $result = GetOptions (
323 'help|h|?' => \$opt_help,
324 'target=s' => \$opt_target,
325 'prefix=s' => \$prefix,
326 'socket-wrapper' => \$opt_socket_wrapper,
327 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
328 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
329 'quick' => \$opt_quick,
331 'immediate' => \$opt_immediate,
332 'expected-failures=s' => \$opt_expected_failures,
333 'exclude=s' => \@opt_exclude,
334 'include=s' => \@opt_include,
335 'srcdir=s' => \$srcdir,
336 'builddir=s' => \$builddir,
337 'verbose' => \$opt_verbose,
338 'testenv' => \$opt_testenv,
340 'analyse-cmd=s' => \$opt_analyse_cmd,
341 'no-lazy-setup' => \$opt_no_lazy_setup,
342 'resetup-environment' => \$opt_resetup_env,
343 'bindir:s' => \$opt_bindir,
344 'format=s' => \$opt_format,
345 'image=s' => \$opt_image,
346 'testlist=s' => \@testlists
349 exit(1) if (not $result);
351 ShowHelp() if ($opt_help);
355 # quick hack to disable rpc validation when using valgrind - its way too slow
356 unless (defined($ENV{VALGRIND})) {
357 $ENV{VALIDATE} = "validate";
358 $ENV{MALLOC_CHECK_} = 2;
361 my $old_pwd = "$RealBin/..";
363 # Backwards compatibility:
364 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
365 if (defined($ENV{FEDORA_DS_ROOT})) {
372 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
375 $torture_maxtime *= 2;
382 die("using an empty prefix isn't allowed") unless $prefix ne "";
384 #Ensure we have the test prefix around
385 mkdir($prefix, 0777) unless -d $prefix;
387 my $prefix_abs = abs_path($prefix);
388 my $srcdir_abs = abs_path($srcdir);
390 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
391 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
393 $ENV{PREFIX} = $prefix;
394 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
395 $ENV{PREFIX_ABS} = $prefix_abs;
396 $ENV{SRCDIR} = $srcdir;
397 $ENV{SRCDIR_ABS} = $srcdir_abs;
399 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
400 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
401 $opt_format = "buildfarm";
404 my $tls_enabled = not $opt_quick;
405 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
406 $ENV{LDB_MODULES_PATH} = "$old_pwd/source4/bin/modules/ldb";
407 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/source4/bin/modules";
408 sub prefix_pathvar($$)
410 my ($name, $newpath) = @_;
411 if (defined($ENV{$name})) {
412 $ENV{$name} = "$newpath:$ENV{$name}";
414 $ENV{$name} = $newpath;
417 prefix_pathvar("PKG_CONFIG_PATH", "$old_pwd/source4/bin/pkgconfig");
418 prefix_pathvar("PYTHONPATH", "$old_pwd/source4/bin/python");
420 if ($opt_socket_wrapper_keep_pcap) {
421 # Socket wrapper keep pcap implies socket wrapper pcap
422 $opt_socket_wrapper_pcap = 1;
425 if ($opt_socket_wrapper_pcap) {
426 # Socket wrapper pcap implies socket wrapper
427 $opt_socket_wrapper = 1;
430 my $socket_wrapper_dir;
431 if ($opt_socket_wrapper) {
432 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
433 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
436 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
441 my $testenv_default = "none";
443 if ($opt_target eq "samba4") {
444 $testenv_default = "member";
445 require target::Samba4;
446 $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
447 } elsif ($opt_target eq "samba3") {
448 my $bindir = ($opt_bindir or "$srcdir/bin");
449 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
450 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
452 $testenv_default = "dc";
453 require target::Samba3;
454 $target = new Samba3($bindir);
455 } elsif ($opt_target eq "win") {
456 die("Windows tests will not run with socket wrapper enabled.")
457 if ($opt_socket_wrapper);
458 $testenv_default = "dc";
459 require target::Windows;
460 $target = new Windows();
461 } elsif ($opt_target eq "kvm") {
462 die("Kvm tests will not run with socket wrapper enabled.")
463 if ($opt_socket_wrapper);
465 die("No image specified") unless ($opt_image);
466 $target = new Kvm($opt_image, undef);
470 # Start a Virtual Distributed Ethernet Switch
471 # Returns the pid of the switch.
473 sub start_vde_switch($)
477 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
479 open(PID, "$path/vde.pid");
487 # Stop a Virtual Distributed Ethernet Switch
488 sub stop_vde_switch($)
494 sub read_test_regexes($)
498 open(LF, "<$name") or die("unable to read $name: $!");
502 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
503 push (@ret, [$1, $4]);
505 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
506 push (@ret, [$_, undef]);
513 if (defined($opt_expected_failures)) {
514 @expected_failures = read_test_regexes($opt_expected_failures);
517 foreach (@opt_exclude) {
518 push (@excludes, read_test_regexes($_));
522 push (@includes, read_test_regexes("samba4-quick"));
525 foreach (@opt_include) {
526 push (@includes, read_test_regexes($_));
529 my $interfaces = join(',', ("127.0.0.6/8",
536 my $conffile = "$prefix_abs/client/client.conf";
537 $ENV{SMB_CONF_PATH} = $conffile;
539 sub write_clientconf($$)
541 my ($conffile, $vars) = @_;
543 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
545 if ( -d "$prefix/client/private" ) {
546 unlink <$prefix/client/private/*>;
548 mkdir("$prefix/client/private", 0777);
551 open(CF, ">$conffile");
552 print CF "[global]\n";
553 if (defined($ENV{VALGRIND})) {
554 print CF "\ticonv:native = true\n";
556 print CF "\ticonv:native = false\n";
558 print CF "\tnetbios name = client\n";
559 if (defined($vars->{DOMAIN})) {
560 print CF "\tworkgroup = $vars->{DOMAIN}\n";
562 if (defined($vars->{REALM})) {
563 print CF "\trealm = $vars->{REALM}\n";
565 if (defined($vars->{NCALRPCDIR})) {
566 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
568 if (defined($vars->{PIDDIR})) {
569 print CF "\tpid directory = $vars->{PIDDIR}\n";
571 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
572 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
574 if ($opt_socket_wrapper) {
575 print CF "\tinterfaces = $interfaces\n";
578 private dir = $prefix_abs/client/private
579 name resolve order = bcast
580 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
582 notify:inotify = false
584 system:anonymous = true
585 torture:basedir = $prefix_abs/client
586 #We don't want to pass our self-tests if the PAC code is wrong
587 gensec:require_pac = true
588 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
595 my $testsdir = "$srcdir/selftest";
597 my %required_envs = ();
604 open(IN, $filename) or die("Unable to open $filename: $!");
607 if ($_ eq "-- TEST --\n") {
614 if (not defined($tests) or $name =~ /$tests/) {
615 $required_envs{$env} = 1;
616 push (@ret, [$name, $env, $cmdline]);
622 close(IN) or die("Error creating recipe");
626 if ($#testlists == -1) {
627 die("No testlists specified");
630 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
631 if ($opt_socket_wrapper) {
632 $ENV{SELFTEST_INTERFACES} = $interfaces;
634 $ENV{SELFTEST_INTERFACES} = "";
637 $ENV{SELFTEST_VERBOSE} = "1";
639 $ENV{SELFTEST_VERBOSE} = "";
642 $ENV{SELFTEST_QUICK} = "1";
644 $ENV{SELFTEST_QUICK} = "";
646 $ENV{SELFTEST_TARGET} = $opt_target;
647 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
648 $ENV{SELFTEST_CONFFILE} = $conffile;
651 foreach my $fn (@testlists) {
652 foreach (read_testlist($fn)) {
654 next if (@includes and not find_in_list(\@includes, $name));
655 push (@available, $_);
660 if ($opt_format eq "buildfarm") {
661 require output::buildfarm;
662 $msg_ops = new output::buildfarm($statistics);
663 } elsif ($opt_format eq "plain") {
664 require output::plain;
665 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
666 } elsif ($opt_format eq "html") {
667 require output::html;
668 mkdir("test-results", 0777);
669 $msg_ops = new output::html("test-results", $statistics);
671 die("Invalid output format '$opt_format'");
675 foreach (@available) {
677 my $skipreason = skip($name);
679 $msg_ops->skip_testsuite($name, $skipreason);
686 print STDERR "No tests to run\n";
690 my $suitestotal = $#todo + 1;
694 my %running_envs = ();
696 my @exported_envvars = (
701 # domain controller stuff
721 "WINBINDD_SOCKET_DIR",
722 "WINBINDD_PRIV_PIPE_DIR"
725 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
727 teardown_env($_) foreach(keys %running_envs);
728 die("Received signal $signame");
736 if ($envname eq "none") {
738 } elsif (defined($running_envs{$envname})) {
739 $testenv_vars = $running_envs{$envname};
740 if (not $target->check_env($testenv_vars)) {
741 $testenv_vars = undef;
744 $testenv_vars = $target->setup_env($envname, $prefix);
747 return undef unless defined($testenv_vars);
749 $running_envs{$envname} = $testenv_vars;
751 SocketWrapper::set_default_iface(6);
752 write_clientconf($conffile, $testenv_vars);
754 foreach (@exported_envvars) {
755 if (defined($testenv_vars->{$_})) {
756 $ENV{$_} = $testenv_vars->{$_};
762 return $testenv_vars;
765 sub exported_envvars_str($)
767 my ($testenv_vars) = @_;
770 foreach (@exported_envvars) {
771 next unless defined($testenv_vars->{$_});
772 $out .= $_."=".$testenv_vars->{$_}."\n";
781 return "" if ($envname eq "none");
782 return $target->getlog_env($running_envs{$envname});
788 return 1 if ($envname eq "none");
789 return $target->check_env($running_envs{$envname});
795 return if ($envname eq "none");
796 $target->teardown_env($running_envs{$envname});
797 delete $running_envs{$envname};
800 if ($opt_no_lazy_setup) {
801 setup_env($_) foreach (keys %required_envs);
805 my $testenv_name = $ENV{SELFTEST_TESTENV};
806 $testenv_name = $testenv_default unless defined($testenv_name);
808 my $testenv_vars = setup_env($testenv_name);
810 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
812 my $envvarstr = exported_envvars_str($testenv_vars);
814 my $term = ($ENV{TERM} or "xterm");
815 system("$term -e 'echo -e \"
816 Welcome to the Samba4 Test environment '$testenv_name'
818 This matches the client environment used in make test
819 server is pid `cat \$PIDDIR/samba.pid`
821 Some useful environment variables:
822 TORTURE_OPTIONS=\$TORTURE_OPTIONS
823 CONFIGURATION=\$CONFIGURATION
826 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
827 teardown_env($testenv_name);
832 $cmd =~ s/([\(\)])/\\$1/g;
834 my $envname = $$_[1];
836 my $envvars = setup_env($envname);
837 if (not defined($envvars)) {
838 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
842 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
845 if (defined($opt_analyse_cmd)) {
846 system("$opt_analyse_cmd \"$name\"");
849 teardown_env($envname) if ($opt_resetup_env);
855 teardown_env($_) foreach (keys %running_envs);
863 # if there were any valgrind failures, show them
864 foreach (<$prefix/valgrind.log*>) {
866 system("grep DWARF2.CFI.reader $_ > /dev/null");
868 print "VALGRIND FAILURE\n";
874 if ($opt_format eq "buildfarm") {
875 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
878 exit $statistics->{SUITES_FAIL};