2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
137 my $opt_target = "samba4";
139 my $opt_socket_wrapper = 0;
140 my $opt_socket_wrapper_pcap = undef;
141 my $opt_socket_wrapper_keep_pcap = undef;
143 my @opt_exclude = ();
144 my @opt_include = ();
146 my $opt_image = undef;
149 my $opt_analyse_cmd = undef;
150 my $opt_resetup_env = undef;
151 my $opt_bindir = undef;
152 my $opt_no_lazy_setup = undef;
165 my ($list, $fullname) = @_;
168 if ($fullname =~ /$$_[0]/) {
169 return ($$_[1]) if ($$_[1]);
181 return find_in_list(\@excludes, $name);
190 return unless ($opt_socket_wrapper_pcap);
191 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
194 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
196 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
198 SocketWrapper::setup_pcap($pcap_file);
205 my ($pcap_file, $exitcode) = @_;
207 return unless ($opt_socket_wrapper_pcap);
208 return if ($opt_socket_wrapper_keep_pcap);
209 return unless ($exitcode == 0);
210 return unless defined($pcap_file);
215 sub run_testsuite($$$$$)
217 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
218 my $pcap_file = setup_pcap($name);
220 Subunit::report_time(time());
221 Subunit::start_testsuite($name);
223 open(RESULTS, "$cmd 2>&1|");
227 TESTS_UNEXPECTED_OK => 0,
228 TESTS_EXPECTED_OK => 0,
229 TESTS_UNEXPECTED_FAIL => 0,
230 TESTS_EXPECTED_FAIL => 0,
235 my $msg_ops = new Subunit::Filter("$name\.", []);
237 parse_results($msg_ops, $statistics, *RESULTS);
241 unless (close(RESULTS)) {
243 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
251 Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
254 my $envlog = getlog_env($envname);
256 print "envlog: $envlog\n";
259 print "command: $cmd\n";
261 my $exitcode = $ret >> 8;
263 Subunit::report_time(time());
264 if ($exitcode == 0) {
265 Subunit::end_testsuite($name, "success");
267 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
270 cleanup_pcap($pcap_file, $exitcode);
272 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
273 print "PCAP FILE: $pcap_file\n";
276 if ($exitcode != 0) {
277 exit(1) if ($opt_one);
285 print "Samba test runner
286 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
287 Copyright (C) Stefan Metzmacher <metze\@samba.org>
289 Usage: $Script [OPTIONS] TESTNAME-REGEX
292 --help this help page
293 --target=samba[34]|win|kvm Samba version to target
294 --testlist=FILE file to read available tests from
297 --prefix=DIR prefix to run tests in [st]
298 --srcdir=DIR source directory [.]
299 --builddir=DIR output directory [.]
300 --exeext=EXT executable extention []
303 --socket-wrapper-pcap save traffic to pcap directories
304 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
306 --socket-wrapper enable socket wrapper
307 --bindir=PATH path to target binaries
310 --ldap=openldap|fedora-ds back samba onto specified ldap server
313 --image=PATH path to KVM image
316 --quick run quick overall test
317 --one abort when the first test fails
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'exclude=s' => \@opt_exclude,
334 'include=s' => \@opt_include,
335 'srcdir=s' => \$srcdir,
336 'builddir=s' => \$builddir,
337 'exeext=s' => \$exeext,
338 'verbose' => \$opt_verbose,
339 'testenv' => \$opt_testenv,
341 'analyse-cmd=s' => \$opt_analyse_cmd,
342 'no-lazy-setup' => \$opt_no_lazy_setup,
343 'resetup-environment' => \$opt_resetup_env,
344 'bindir:s' => \$opt_bindir,
345 'image=s' => \$opt_image,
346 'testlist=s' => \@testlists
349 exit(1) if (not $result);
351 ShowHelp() if ($opt_help);
355 # quick hack to disable rpc validation when using valgrind - its way too slow
356 unless (defined($ENV{VALGRIND})) {
357 $ENV{VALIDATE} = "validate";
358 $ENV{MALLOC_CHECK_} = 2;
361 my $bindir = ($opt_bindir or "$builddir/bin");
362 my $bindir_abs = abs_path($bindir);
364 # Backwards compatibility:
365 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
366 if (defined($ENV{FEDORA_DS_ROOT})) {
373 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
376 $torture_maxtime *= 2;
383 die("using an empty prefix isn't allowed") unless $prefix ne "";
385 #Ensure we have the test prefix around
386 mkdir($prefix, 0777) unless -d $prefix;
388 my $prefix_abs = abs_path($prefix);
389 my $srcdir_abs = abs_path($srcdir);
390 my $builddir_abs = abs_path($builddir);
392 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
393 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
395 $ENV{PREFIX} = $prefix;
396 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
397 $ENV{PREFIX_ABS} = $prefix_abs;
398 $ENV{SRCDIR} = $srcdir;
399 $ENV{SRCDIR_ABS} = $srcdir_abs;
400 $ENV{BUILDDIR} = $builddir;
401 $ENV{BUILDDIR_ABS} = $builddir_abs;
402 $ENV{EXEEXT} = $exeext;
404 my $tls_enabled = not $opt_quick;
405 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
406 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
407 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
408 sub prefix_pathvar($$)
410 my ($name, $newpath) = @_;
411 if (defined($ENV{$name})) {
412 $ENV{$name} = "$newpath:$ENV{$name}";
414 $ENV{$name} = $newpath;
417 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
418 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
420 if ($opt_socket_wrapper_keep_pcap) {
421 # Socket wrapper keep pcap implies socket wrapper pcap
422 $opt_socket_wrapper_pcap = 1;
425 if ($opt_socket_wrapper_pcap) {
426 # Socket wrapper pcap implies socket wrapper
427 $opt_socket_wrapper = 1;
430 my $socket_wrapper_dir;
431 if ($opt_socket_wrapper) {
432 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
433 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
436 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
441 my $testenv_default = "none";
443 if ($opt_target eq "samba4") {
444 $testenv_default = "member";
445 require target::Samba4;
446 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
447 } elsif ($opt_target eq "samba3") {
448 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
449 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
451 $testenv_default = "member";
452 require target::Samba3;
453 $target = new Samba3($bindir);
454 } elsif ($opt_target eq "win") {
455 die("Windows tests will not run with socket wrapper enabled.")
456 if ($opt_socket_wrapper);
457 $testenv_default = "dc";
458 require target::Windows;
459 $target = new Windows();
460 } elsif ($opt_target eq "kvm") {
461 die("Kvm tests will not run with socket wrapper enabled.")
462 if ($opt_socket_wrapper);
464 die("No image specified") unless ($opt_image);
465 $target = new Kvm($opt_image, undef);
469 # Start a Virtual Distributed Ethernet Switch
470 # Returns the pid of the switch.
472 sub start_vde_switch($)
476 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
478 open(PID, "$path/vde.pid");
486 # Stop a Virtual Distributed Ethernet Switch
487 sub stop_vde_switch($)
493 sub read_test_regexes($)
497 open(LF, "<$name") or die("unable to read $name: $!");
501 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
502 push (@ret, [$1, $4]);
504 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
505 push (@ret, [$_, undef]);
512 foreach (@opt_exclude) {
513 push (@excludes, read_test_regexes($_));
516 foreach (@opt_include) {
517 push (@includes, read_test_regexes($_));
520 my $interfaces = join(',', ("127.0.0.6/8",
527 my $conffile = "$prefix_abs/client/client.conf";
528 $ENV{SMB_CONF_PATH} = $conffile;
530 sub write_clientconf($$)
532 my ($conffile, $vars) = @_;
534 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
536 if ( -d "$prefix/client/private" ) {
537 unlink <$prefix/client/private/*>;
539 mkdir("$prefix/client/private", 0777);
542 if ( -d "$prefix/client/lock" ) {
543 unlink <$prefix/client/lockdir/*>;
545 mkdir("$prefix/client/lockdir", 0777);
548 open(CF, ">$conffile");
549 print CF "[global]\n";
550 if (defined($ENV{VALGRIND})) {
551 print CF "\ticonv:native = true\n";
553 print CF "\ticonv:native = false\n";
555 print CF "\tnetbios name = client\n";
556 if (defined($vars->{DOMAIN})) {
557 print CF "\tworkgroup = $vars->{DOMAIN}\n";
559 if (defined($vars->{REALM})) {
560 print CF "\trealm = $vars->{REALM}\n";
562 if ($opt_socket_wrapper) {
563 print CF "\tinterfaces = $interfaces\n";
566 private dir = $prefix_abs/client/private
567 lock dir = $prefix_abs/client/lockdir
568 name resolve order = bcast
569 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
571 notify:inotify = false
573 system:anonymous = true
574 client lanman auth = Yes
575 torture:basedir = $prefix_abs/client
576 #We don't want to pass our self-tests if the PAC code is wrong
577 gensec:require_pac = true
578 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
585 my $testsdir = "$srcdir/selftest";
587 my %required_envs = ();
594 open(IN, $filename) or die("Unable to open $filename: $!");
597 if ($_ eq "-- TEST --\n") {
604 if (not defined($tests) or $name =~ /$tests/) {
605 $required_envs{$env} = 1;
606 push (@ret, [$name, $env, $cmdline]);
612 close(IN) or die("Error creating recipe");
616 if ($#testlists == -1) {
617 die("No testlists specified");
620 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
621 if ($opt_socket_wrapper) {
622 $ENV{SELFTEST_INTERFACES} = $interfaces;
624 $ENV{SELFTEST_INTERFACES} = "";
627 $ENV{SELFTEST_VERBOSE} = "1";
629 $ENV{SELFTEST_VERBOSE} = "";
632 $ENV{SELFTEST_QUICK} = "1";
634 $ENV{SELFTEST_QUICK} = "";
636 $ENV{SELFTEST_TARGET} = $opt_target;
637 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
640 foreach my $fn (@testlists) {
641 foreach (read_testlist($fn)) {
643 next if (@includes and not defined(find_in_list(\@includes, $name)));
644 push (@available, $_);
648 Subunit::testsuite_count($#available+1);
649 Subunit::report_time(time());
651 foreach (@available) {
653 my $skipreason = skip($name);
654 if (defined($skipreason)) {
655 Subunit::skip_testsuite($name, $skipreason);
662 print STDERR "No tests to run\n";
666 my $suitestotal = $#todo + 1;
670 my %running_envs = ();
672 sub get_running_env($)
680 return $running_envs{$envname};
683 my @exported_envvars = (
688 # domain controller stuff
708 "WINBINDD_SOCKET_DIR",
709 "WINBINDD_PRIV_PIPE_DIR"
712 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
714 teardown_env($_) foreach(keys %running_envs);
715 die("Received signal $signame");
722 my $testenv_vars = undef;
728 $option =~ s/^[^:]*//;
731 $option = "client" if $option eq "";
733 if ($envname eq "none") {
735 } elsif (defined(get_running_env($envname))) {
736 $testenv_vars = get_running_env($envname);
737 if (not $target->check_env($testenv_vars)) {
738 $testenv_vars = undef;
741 $testenv_vars = $target->setup_env($envname, $prefix);
744 return undef unless defined($testenv_vars);
746 $running_envs{$envname} = $testenv_vars;
748 if ($option eq "local") {
749 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
750 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
751 } elsif ($option eq "client") {
752 SocketWrapper::set_default_iface(6);
753 write_clientconf($conffile, $testenv_vars);
754 $ENV{SMB_CONF_PATH} = $conffile;
756 die("Unknown option[$option] for envname[$envname]");
759 foreach (@exported_envvars) {
760 if (defined($testenv_vars->{$_})) {
761 $ENV{$_} = $testenv_vars->{$_};
767 return $testenv_vars;
770 sub exported_envvars_str($)
772 my ($testenv_vars) = @_;
775 foreach (@exported_envvars) {
776 next unless defined($testenv_vars->{$_});
777 $out .= $_."=".$testenv_vars->{$_}."\n";
786 return "" if ($envname eq "none");
787 return $target->getlog_env(get_running_env($envname));
793 return 1 if ($envname eq "none");
794 return $target->check_env(get_running_env($envname));
800 return if ($envname eq "none");
801 $target->teardown_env(get_running_env($envname));
802 delete $running_envs{$envname};
805 if ($opt_no_lazy_setup) {
806 setup_env($_) foreach (keys %required_envs);
810 my $testenv_name = $ENV{SELFTEST_TESTENV};
811 $testenv_name = $testenv_default unless defined($testenv_name);
813 my $testenv_vars = setup_env($testenv_name);
815 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
817 my $envvarstr = exported_envvars_str($testenv_vars);
819 my $term = ($ENV{TERM} or "xterm");
820 system("$term -e 'echo -e \"
821 Welcome to the Samba4 Test environment '$testenv_name'
823 This matches the client environment used in make test
824 server is pid `cat \$PIDDIR/samba.pid`
826 Some useful environment variables:
827 TORTURE_OPTIONS=\$TORTURE_OPTIONS
828 SMB_CONF_PATH=\$SMB_CONF_PATH
831 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
832 teardown_env($testenv_name);
837 $cmd =~ s/([\(\)])/\\$1/g;
839 my $envname = $$_[1];
841 my $envvars = setup_env($envname);
842 if (not defined($envvars)) {
843 Subunit::skip_testsuite($name,
844 "unable to set up environment $envname");
848 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
850 if (defined($opt_analyse_cmd)) {
851 system("$opt_analyse_cmd \"$name\"");
854 teardown_env($envname) if ($opt_resetup_env);
860 teardown_env($_) foreach (keys %running_envs);
866 # if there were any valgrind failures, show them
867 foreach (<$prefix/valgrind.log*>) {
869 system("grep DWARF2.CFI.reader $_ > /dev/null");
871 print "VALGRIND FAILURE\n";