2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win|kvm>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME.TEST-NAME
72 The reason for a test can also be specified, by adding a hash sign (#) and the reason
77 Specify a file containing a list of tests that should be skipped. Possible
78 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
79 for the --expected-failures flag.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
136 my $opt_target = "samba4";
138 my $opt_socket_wrapper = 0;
139 my $opt_socket_wrapper_pcap = undef;
140 my $opt_socket_wrapper_keep_pcap = undef;
142 my $opt_immediate = 0;
143 my $opt_expected_failures = undef;
144 my @opt_exclude = ();
145 my @opt_include = ();
147 my $opt_image = undef;
150 my $opt_analyse_cmd = undef;
151 my $opt_resetup_env = undef;
152 my $opt_bindir = undef;
153 my $opt_no_lazy_setup = undef;
154 my $opt_format = "plain";
161 my @expected_failures = ();
168 TESTS_UNEXPECTED_OK => 0,
169 TESTS_EXPECTED_OK => 0,
170 TESTS_UNEXPECTED_FAIL => 0,
171 TESTS_EXPECTED_FAIL => 0,
178 my ($list, $fullname) = @_;
181 if ($fullname =~ /$$_[0]/) {
182 return ($$_[1]) if ($$_[1]);
183 return "NO REASON SPECIFIED";
190 sub expecting_failure($)
193 return find_in_list(\@expected_failures, $name);
200 return find_in_list(\@excludes, $name);
209 return unless ($opt_socket_wrapper_pcap);
210 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
213 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
215 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
217 SocketWrapper::setup_pcap($pcap_file);
222 sub cleanup_pcap($$$)
224 my ($pcap_file, $expected_ret, $ret) = @_;
226 return unless ($opt_socket_wrapper_pcap);
227 return if ($opt_socket_wrapper_keep_pcap);
228 return unless ($expected_ret == $ret);
229 return unless defined($pcap_file);
234 sub run_testsuite($$$$$$)
236 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
237 my $pcap_file = setup_pcap($name);
239 $msg_ops->start_test([], $name);
241 unless (open(RESULT, "$cmd 2>&1|")) {
242 $statistics->{TESTS_ERROR}++;
243 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
244 $statistics->{SUITES_FAIL}++;
248 my $expected_ret = parse_results(
249 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
251 my $envlog = getlog_env($envname);
252 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
254 $msg_ops->output_msg("CMD: $cmd\n");
256 my $ret = close(RESULT);
257 $ret = 0 unless $ret == 1;
259 my $exitcode = $? >> 8;
262 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
264 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
267 cleanup_pcap($pcap_file, $expected_ret, $ret);
269 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
270 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
273 if ($ret != $expected_ret) {
274 $statistics->{SUITES_FAIL}++;
275 exit(1) if ($opt_one);
278 return ($ret == $expected_ret);
283 print "Samba test runner
284 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
286 Usage: $Script [OPTIONS] TESTNAME-REGEX
289 --help this help page
290 --target=samba[34]|win|kvm Samba version to target
291 --testlist=FILE file to read available tests from
294 --prefix=DIR prefix to run tests in [st]
295 --srcdir=DIR source directory [.]
296 --builddir=DIR output directory [.]
299 --socket-wrapper-pcap save traffic to pcap directories
300 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
302 --socket-wrapper enable socket wrapper
303 --expected-failures=FILE specify list of tests that is guaranteed to fail
306 --ldap=openldap|fedora-ds back samba onto specified ldap server
309 --bindir=PATH path to binaries
312 --image=PATH path to KVM image
315 --quick run quick overall test
316 --one abort when the first test fails
317 --immediate print test output for failed tests during run
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'immediate' => \$opt_immediate,
334 'expected-failures=s' => \$opt_expected_failures,
335 'exclude=s' => \@opt_exclude,
336 'include=s' => \@opt_include,
337 'srcdir=s' => \$srcdir,
338 'builddir=s' => \$builddir,
339 'verbose' => \$opt_verbose,
340 'testenv' => \$opt_testenv,
342 'analyse-cmd=s' => \$opt_analyse_cmd,
343 'no-lazy-setup' => \$opt_no_lazy_setup,
344 'resetup-environment' => \$opt_resetup_env,
345 'bindir:s' => \$opt_bindir,
346 'format=s' => \$opt_format,
347 'image=s' => \$opt_image,
348 'testlist=s' => \@testlists
351 exit(1) if (not $result);
353 ShowHelp() if ($opt_help);
357 # quick hack to disable rpc validation when using valgrind - its way too slow
358 unless (defined($ENV{VALGRIND})) {
359 $ENV{VALIDATE} = "validate";
360 $ENV{MALLOC_CHECK_} = 2;
363 my $old_pwd = "$RealBin/..";
365 # Backwards compatibility:
366 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
367 if (defined($ENV{FEDORA_DS_ROOT})) {
374 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
377 $torture_maxtime *= 2;
384 die("using an empty prefix isn't allowed") unless $prefix ne "";
386 #Ensure we have the test prefix around
387 mkdir($prefix, 0777) unless -d $prefix;
389 my $prefix_abs = abs_path($prefix);
390 my $srcdir_abs = abs_path($srcdir);
392 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
393 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
395 $ENV{PREFIX} = $prefix;
396 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
397 $ENV{PREFIX_ABS} = $prefix_abs;
398 $ENV{SRCDIR} = $srcdir;
399 $ENV{SRCDIR_ABS} = $srcdir_abs;
401 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
402 ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
403 $opt_format = "buildfarm";
406 my $tls_enabled = not $opt_quick;
407 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
408 $ENV{LDB_MODULES_PATH} = "$old_pwd/source4/bin/modules/ldb";
409 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/source4/bin/modules";
410 sub prefix_pathvar($$)
412 my ($name, $newpath) = @_;
413 if (defined($ENV{$name})) {
414 $ENV{$name} = "$newpath:$ENV{$name}";
416 $ENV{$name} = $newpath;
419 prefix_pathvar("PKG_CONFIG_PATH", "$old_pwd/source4/bin/pkgconfig");
420 prefix_pathvar("PYTHONPATH", "$old_pwd/source4/bin/python");
422 if ($opt_socket_wrapper_keep_pcap) {
423 # Socket wrapper keep pcap implies socket wrapper pcap
424 $opt_socket_wrapper_pcap = 1;
427 if ($opt_socket_wrapper_pcap) {
428 # Socket wrapper pcap implies socket wrapper
429 $opt_socket_wrapper = 1;
432 my $socket_wrapper_dir;
433 if ($opt_socket_wrapper) {
434 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
435 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
437 warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
441 my $testenv_default = "none";
443 if ($opt_target eq "samba4") {
444 $testenv_default = "member";
445 require target::Samba4;
446 $target = new Samba4($opt_bindir or "$srcdir/bin", $ldap, "$srcdir/setup");
447 } elsif ($opt_target eq "samba3") {
448 if ($opt_socket_wrapper and `$opt_bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
449 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
451 $testenv_default = "dc";
452 require target::Samba3;
453 $target = new Samba3($opt_bindir);
454 } elsif ($opt_target eq "win") {
455 die("Windows tests will not run with socket wrapper enabled.")
456 if ($opt_socket_wrapper);
457 $testenv_default = "dc";
458 require target::Windows;
459 $target = new Windows();
460 } elsif ($opt_target eq "kvm") {
461 die("Kvm tests will not run with socket wrapper enabled.")
462 if ($opt_socket_wrapper);
464 die("No image specified") unless ($opt_image);
465 $target = new Kvm($opt_image, undef);
469 # Start a Virtual Distributed Ethernet Switch
470 # Returns the pid of the switch.
472 sub start_vde_switch($)
476 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
478 open(PID, "$path/vde.pid");
486 # Stop a Virtual Distributed Ethernet Switch
487 sub stop_vde_switch($)
493 sub read_test_regexes($)
497 open(LF, "<$name") or die("unable to read $name: $!");
501 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
502 push (@ret, [$1, $4]);
504 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
505 push (@ret, [$_, undef]);
512 if (defined($opt_expected_failures)) {
513 @expected_failures = read_test_regexes($opt_expected_failures);
516 foreach (@opt_exclude) {
517 push (@excludes, read_test_regexes($_));
521 push (@includes, read_test_regexes("samba4-quick"));
524 foreach (@opt_include) {
525 push (@includes, read_test_regexes($_));
528 my $interfaces = join(',', ("127.0.0.6/8",
535 my $conffile = "$prefix_abs/client/client.conf";
536 $ENV{SMB_CONF_PATH} = $conffile;
538 sub write_clientconf($$)
540 my ($conffile, $vars) = @_;
542 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
544 if ( -d "$prefix/client/private" ) {
545 unlink <$prefix/client/private/*>;
547 mkdir("$prefix/client/private", 0777);
550 open(CF, ">$conffile");
551 print CF "[global]\n";
552 if (defined($ENV{VALGRIND})) {
553 print CF "\ticonv:native = true\n";
555 print CF "\ticonv:native = false\n";
557 print CF "\tnetbios name = client\n";
558 if (defined($vars->{DOMAIN})) {
559 print CF "\tworkgroup = $vars->{DOMAIN}\n";
561 if (defined($vars->{REALM})) {
562 print CF "\trealm = $vars->{REALM}\n";
564 if (defined($vars->{NCALRPCDIR})) {
565 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
567 if (defined($vars->{PIDDIR})) {
568 print CF "\tpid directory = $vars->{PIDDIR}\n";
570 if (defined($vars->{WINBINDD_SOCKET_DIR})) {
571 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
573 if ($opt_socket_wrapper) {
574 print CF "\tinterfaces = $interfaces\n";
577 private dir = $prefix_abs/client/private
578 name resolve order = bcast
579 panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
581 notify:inotify = false
583 system:anonymous = true
584 torture:basedir = $prefix_abs/client
585 #We don't want to pass our self-tests if the PAC code is wrong
586 gensec:require_pac = true
587 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
594 my $testsdir = "$srcdir/selftest";
596 my %required_envs = ();
603 open(IN, $filename) or die("Unable to open $filename: $!");
606 if ($_ eq "-- TEST --\n") {
613 if (not defined($tests) or $name =~ /$tests/) {
614 $required_envs{$env} = 1;
615 push (@ret, [$name, $env, $cmdline]);
621 close(IN) or die("Error creating recipe");
625 if ($#testlists == -1) {
626 die("No testlists specified");
629 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
630 if ($opt_socket_wrapper) {
631 $ENV{SELFTEST_INTERFACES} = $interfaces;
633 $ENV{SELFTEST_INTERFACES} = "";
636 $ENV{SELFTEST_VERBOSE} = "1";
638 $ENV{SELFTEST_VERBOSE} = "";
641 $ENV{SELFTEST_QUICK} = "1";
643 $ENV{SELFTEST_QUICK} = "";
645 $ENV{SELFTEST_TARGET} = $opt_target;
646 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
647 $ENV{SELFTEST_CONFFILE} = $conffile;
650 foreach my $fn (@testlists) {
651 foreach (read_testlist($fn)) {
653 next if (@includes and not find_in_list(\@includes, $name));
654 push (@available, $_);
659 if ($opt_format eq "buildfarm") {
660 require output::buildfarm;
661 $msg_ops = new output::buildfarm($statistics);
662 } elsif ($opt_format eq "plain") {
663 require output::plain;
664 $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
665 } elsif ($opt_format eq "html") {
666 require output::html;
667 mkdir("test-results", 0777);
668 $msg_ops = new output::html("test-results", $statistics);
670 die("Invalid output format '$opt_format'");
674 foreach (@available) {
676 my $skipreason = skip($name);
678 $msg_ops->skip_testsuite($name, $skipreason);
685 print STDERR "No tests to run\n";
689 my $suitestotal = $#todo + 1;
693 my %running_envs = ();
695 my @exported_envvars = (
700 # domain controller stuff
720 "WINBINDD_SOCKET_DIR",
721 "WINBINDD_PRIV_PIPE_DIR"
724 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
726 teardown_env($_) foreach(keys %running_envs);
727 die("Received signal $signame");
735 if ($envname eq "none") {
737 } elsif (defined($running_envs{$envname})) {
738 $testenv_vars = $running_envs{$envname};
739 if (not $target->check_env($testenv_vars)) {
740 $testenv_vars = undef;
743 $testenv_vars = $target->setup_env($envname, $prefix);
746 return undef unless defined($testenv_vars);
748 $running_envs{$envname} = $testenv_vars;
750 SocketWrapper::set_default_iface(6);
751 write_clientconf($conffile, $testenv_vars);
753 foreach (@exported_envvars) {
754 if (defined($testenv_vars->{$_})) {
755 $ENV{$_} = $testenv_vars->{$_};
761 return $testenv_vars;
764 sub exported_envvars_str($)
766 my ($testenv_vars) = @_;
769 foreach (@exported_envvars) {
770 next unless defined($testenv_vars->{$_});
771 $out .= $_."=".$testenv_vars->{$_}."\n";
780 return "" if ($envname eq "none");
781 return $target->getlog_env($running_envs{$envname});
787 return 1 if ($envname eq "none");
788 return $target->check_env($running_envs{$envname});
794 return if ($envname eq "none");
795 $target->teardown_env($running_envs{$envname});
796 delete $running_envs{$envname};
799 if ($opt_no_lazy_setup) {
800 setup_env($_) foreach (keys %required_envs);
804 my $testenv_name = $ENV{SELFTEST_TESTENV};
805 $testenv_name = $testenv_default unless defined($testenv_name);
807 my $testenv_vars = setup_env($testenv_name);
809 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
811 my $envvarstr = exported_envvars_str($testenv_vars);
813 my $term = ($ENV{TERM} or "xterm");
814 system("$term -e 'echo -e \"
815 Welcome to the Samba4 Test environment '$testenv_name'
817 This matches the client environment used in make test
818 server is pid `cat \$PIDDIR/samba.pid`
820 Some useful environment variables:
821 TORTURE_OPTIONS=\$TORTURE_OPTIONS
822 CONFIGURATION=\$CONFIGURATION
825 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
826 teardown_env($testenv_name);
831 $cmd =~ s/([\(\)])/\\$1/g;
833 my $envname = $$_[1];
835 my $envvars = setup_env($envname);
836 if (not defined($envvars)) {
837 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
841 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
844 if (defined($opt_analyse_cmd)) {
845 system("$opt_analyse_cmd \"$name\"");
848 teardown_env($envname) if ($opt_resetup_env);
854 teardown_env($_) foreach (keys %running_envs);
862 # if there were any valgrind failures, show them
863 foreach (<$prefix/valgrind.log*>) {
865 system("grep DWARF2.CFI.reader $_ > /dev/null");
867 print "VALGRIND FAILURE\n";
873 if ($opt_format eq "buildfarm") {
874 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
877 exit $statistics->{SUITES_FAIL};