2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd qw(abs_path);
140 use Time::HiRes qw(time);
144 my $opt_target = "samba4";
146 my $opt_socket_wrapper = 0;
147 my $opt_socket_wrapper_pcap = undef;
148 my $opt_socket_wrapper_keep_pcap = undef;
150 my @opt_exclude = ();
151 my @opt_include = ();
153 my $opt_image = undef;
156 my $opt_resetup_env = undef;
157 my $opt_bindir = undef;
158 my $opt_load_list = undef;
171 my ($list, $fullname) = @_;
174 if ($fullname =~ /$$_[0]/) {
175 return ($$_[1]) if ($$_[1]);
187 return find_in_list(\@excludes, $name);
196 return unless ($opt_socket_wrapper_pcap);
197 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
200 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
202 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
204 SocketWrapper::setup_pcap($pcap_file);
211 my ($pcap_file, $exitcode) = @_;
213 return unless ($opt_socket_wrapper_pcap);
214 return if ($opt_socket_wrapper_keep_pcap);
215 return unless ($exitcode == 0);
216 return unless defined($pcap_file);
221 # expand strings from %ENV
222 sub expand_environment_strings($)
225 # we use a reverse sort so we do the longer ones first
226 foreach my $k (sort { $b cmp $a } keys %ENV) {
227 $s =~ s/\$$k/$ENV{$k}/g;
232 sub run_testsuite($$$$$)
234 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
235 my $pcap_file = setup_pcap($name);
237 Subunit::start_testsuite($name);
238 Subunit::progress_push();
239 Subunit::report_time(time());
241 Subunit::report_time(time());
242 Subunit::progress_pop();
245 Subunit::progress_pop();
246 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
249 Subunit::end_testsuite($name, "error",
250 sprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127), ($? & 128) ? 'with' : 'without'));
254 my $exitcode = $? >> 8;
256 my $envlog = getlog_env($envname);
258 print "envlog: $envlog\n";
261 print "command: $cmd\n";
262 printf "expanded command: %s\n", expand_environment_strings($cmd);
264 if ($exitcode == 0) {
265 Subunit::end_testsuite($name, "success");
267 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
270 cleanup_pcap($pcap_file, $exitcode);
272 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
273 print "PCAP FILE: $pcap_file\n";
276 if ($exitcode != 0) {
277 exit(1) if ($opt_one);
285 print "Samba test runner
286 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
287 Copyright (C) Stefan Metzmacher <metze\@samba.org>
289 Usage: $Script [OPTIONS] TESTNAME-REGEX
292 --help this help page
293 --target=samba[34]|win|kvm Samba version to target
294 --testlist=FILE file to read available tests from
297 --prefix=DIR prefix to run tests in [st]
298 --srcdir=DIR source directory [.]
299 --builddir=DIR output directory [.]
300 --exeext=EXT executable extention []
303 --socket-wrapper-pcap save traffic to pcap directories
304 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
306 --socket-wrapper enable socket wrapper
307 --bindir=PATH path to target binaries
310 --ldap=openldap|fedora-ds back samba onto specified ldap server
313 --image=PATH path to KVM image
316 --quick run quick overall test
317 --one abort when the first test fails
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'exclude=s' => \@opt_exclude,
334 'include=s' => \@opt_include,
335 'srcdir=s' => \$srcdir,
336 'builddir=s' => \$builddir,
337 'exeext=s' => \$exeext,
338 'verbose' => \$opt_verbose,
339 'testenv' => \$opt_testenv,
341 'resetup-environment' => \$opt_resetup_env,
342 'bindir:s' => \$opt_bindir,
343 'image=s' => \$opt_image,
344 'testlist=s' => \@testlists,
345 'load-list=s' => \$opt_load_list,
348 exit(1) if (not $result);
350 ShowHelp() if ($opt_help);
352 # we want unbuffered output
357 # quick hack to disable rpc validation when using valgrind - its way too slow
358 unless (defined($ENV{VALGRIND})) {
359 $ENV{VALIDATE} = "validate";
360 $ENV{MALLOC_CHECK_} = 2;
363 # make all our python scripts unbuffered
364 $ENV{PYTHONUNBUFFERED} = 1;
366 my $bindir = ($opt_bindir or "$builddir/bin");
367 my $bindir_abs = abs_path($bindir);
369 # Backwards compatibility:
370 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
371 if (defined($ENV{FEDORA_DS_ROOT})) {
378 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
381 $torture_maxtime *= 2;
388 die("using an empty prefix isn't allowed") unless $prefix ne "";
390 #Ensure we have the test prefix around
391 mkdir($prefix, 0777) unless -d $prefix;
393 my $prefix_abs = abs_path($prefix);
394 my $srcdir_abs = abs_path($srcdir);
395 my $builddir_abs = abs_path($builddir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX} = $prefix;
401 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS} = $prefix_abs;
403 $ENV{SRCDIR} = $srcdir;
404 $ENV{SRCDIR_ABS} = $srcdir_abs;
405 $ENV{BUILDDIR} = $builddir;
406 $ENV{BUILDDIR_ABS} = $builddir_abs;
407 $ENV{EXEEXT} = $exeext;
409 my $tls_enabled = not $opt_quick;
410 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
411 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
412 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
413 sub prefix_pathvar($$)
415 my ($name, $newpath) = @_;
416 if (defined($ENV{$name})) {
417 $ENV{$name} = "$newpath:$ENV{$name}";
419 $ENV{$name} = $newpath;
422 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
423 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
425 if ($opt_socket_wrapper_keep_pcap) {
426 # Socket wrapper keep pcap implies socket wrapper pcap
427 $opt_socket_wrapper_pcap = 1;
430 if ($opt_socket_wrapper_pcap) {
431 # Socket wrapper pcap implies socket wrapper
432 $opt_socket_wrapper = 1;
435 my $socket_wrapper_dir;
436 if ($opt_socket_wrapper) {
437 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
438 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
441 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
446 my $testenv_default = "none";
448 if ($opt_target eq "samba4") {
449 $testenv_default = "all";
450 require target::Samba4;
451 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
452 } elsif ($opt_target eq "samba3") {
453 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
454 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
456 $testenv_default = "member";
457 require target::Samba3;
458 $target = new Samba3($bindir);
459 } elsif ($opt_target eq "win") {
460 die("Windows tests will not run with socket wrapper enabled.")
461 if ($opt_socket_wrapper);
462 $testenv_default = "dc";
463 require target::Windows;
464 $target = new Windows();
465 } elsif ($opt_target eq "kvm") {
466 die("Kvm tests will not run with socket wrapper enabled.")
467 if ($opt_socket_wrapper);
469 die("No image specified") unless ($opt_image);
470 $target = new Kvm($opt_image, undef);
474 # Start a Virtual Distributed Ethernet Switch
475 # Returns the pid of the switch.
477 sub start_vde_switch($)
481 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
483 open(PID, "$path/vde.pid");
491 # Stop a Virtual Distributed Ethernet Switch
492 sub stop_vde_switch($)
498 sub read_test_regexes($)
502 open(LF, "<$name") or die("unable to read $name: $!");
506 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
507 push (@ret, [$1, $4]);
509 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
510 push (@ret, [$_, undef]);
517 foreach (@opt_exclude) {
518 push (@excludes, read_test_regexes($_));
521 foreach (@opt_include) {
522 push (@includes, read_test_regexes($_));
525 my $interfaces = join(',', ("127.0.0.11/8",
532 my $conffile = "$prefix_abs/client/client.conf";
533 $ENV{SMB_CONF_PATH} = $conffile;
535 sub write_clientconf($$)
537 my ($conffile, $vars) = @_;
539 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
541 if ( -d "$prefix/client/private" ) {
542 unlink <$prefix/client/private/*>;
544 mkdir("$prefix/client/private", 0777);
547 if ( -d "$prefix/client/lockdir" ) {
548 unlink <$prefix/client/lockdir/*>;
550 mkdir("$prefix/client/lockdir", 0777);
553 if ( -d "$prefix_abs/client/ncalrpcdir" ) {
554 unlink <$prefix/client/ncalrpcdir/*>;
556 mkdir("$prefix/client/ncalrpcdir", 0777);
559 open(CF, ">$conffile");
560 print CF "[global]\n";
561 if (defined($ENV{VALGRIND})) {
562 print CF "\ticonv:native = true\n";
564 print CF "\ticonv:native = false\n";
566 print CF "\tnetbios name = client\n";
567 if (defined($vars->{DOMAIN})) {
568 print CF "\tworkgroup = $vars->{DOMAIN}\n";
570 if (defined($vars->{REALM})) {
571 print CF "\trealm = $vars->{REALM}\n";
573 if ($opt_socket_wrapper) {
574 print CF "\tinterfaces = $interfaces\n";
577 private dir = $prefix_abs/client/private
578 lock dir = $prefix_abs/client/lockdir
579 ncalrpc dir = $prefix_abs/client/ncalrpcdir
580 name resolve order = bcast file
581 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
583 notify:inotify = false
585 system:anonymous = true
586 client lanman auth = Yes
588 torture:basedir = $prefix_abs/client
589 #We don't want to pass our self-tests if the PAC code is wrong
590 gensec:require_pac = true
591 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
592 setup directory = ./setup
593 resolv:host file = $prefix_abs/dns_host_file
600 my $testsdir = "$srcdir/selftest";
602 sub should_run_test($)
608 for (my $i=0; $i <= $#tests; $i++) {
609 if ($name =~ /$tests[$i]/i) {
621 open(IN, $filename) or die("Unable to open $filename: $!");
624 if (/-- TEST(-LOADLIST|-IDLIST|) --\n/) {
625 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
626 my $supports_idlist = (defined($1) and $1 eq "-IDLIST");
633 if (should_run_test($name) == 1) {
634 push (@ret, [$name, $env, $cmdline, $supports_loadlist, $supports_idlist]);
640 close(IN) or die("Error creating recipe");
644 if ($#testlists == -1) {
645 die("No testlists specified");
648 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
649 if ($opt_socket_wrapper) {
650 $ENV{SELFTEST_INTERFACES} = $interfaces;
652 $ENV{SELFTEST_INTERFACES} = "";
655 $ENV{SELFTEST_VERBOSE} = "1";
657 $ENV{SELFTEST_VERBOSE} = "";
660 $ENV{SELFTEST_QUICK} = "1";
662 $ENV{SELFTEST_QUICK} = "";
664 $ENV{SELFTEST_TARGET} = $opt_target;
665 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
668 foreach my $fn (@testlists) {
669 foreach (read_testlist($fn)) {
671 next if (@includes and not defined(find_in_list(\@includes, $name)));
672 push (@available, $_);
676 my $restricted = undef;
677 my $restricted_used = {};
679 if ($opt_load_list) {
681 open(LOAD_LIST, "<$opt_load_list") or die("Unable to open $opt_load_list");
682 while (<LOAD_LIST>) {
684 push (@$restricted, $_);
689 Subunit::progress($#available+1);
690 Subunit::report_time(time());
692 my $individual_tests = undef;
693 $individual_tests = {};
695 foreach my $testsuite (@available) {
696 my $name = $$testsuite[0];
697 my $skipreason = skip($name);
698 if (defined($skipreason)) {
699 Subunit::skip_testsuite($name, $skipreason);
700 } elsif (defined($restricted)) {
701 # Find the testsuite for this test
703 foreach my $r (@$restricted) {
705 $individual_tests->{$name} = [];
707 $restricted_used->{$r} = 1;
708 } elsif (substr($r, 0, length($name)+1) eq "$name.") {
709 push(@{$individual_tests->{$name}}, $r);
711 $restricted_used->{$r} = 1;
714 push(@todo, $testsuite) if ($match);
716 push(@todo, $testsuite);
720 if (defined($restricted)) {
721 foreach (@$restricted) {
722 unless (defined($restricted_used->{$_})) {
723 print "No test or testsuite found matching $_\n";
726 } elsif ($#todo == -1) {
727 print STDERR "No tests to run\n";
731 my $suitestotal = $#todo + 1;
735 my %running_envs = ();
737 sub get_running_env($)
745 return $running_envs{$envname};
748 my @exported_envvars = (
753 # domain controller stuff
762 "MEMBER_NETBIOSNAME",
763 "MEMBER_NETBIOSALIAS",
765 # rpc proxy controller stuff
767 "RPC_PROXY_SERVER_IP",
768 "RPC_PROXY_NETBIOSNAME",
769 "RPC_PROXY_NETBIOSALIAS",
771 # domain controller stuff for Vampired DC
773 "VAMPIRE_DC_SERVER_IP",
774 "VAMPIRE_DC_NETBIOSNAME",
775 "VAMPIRE_DC_NETBIOSALIAS",
792 "WINBINDD_SOCKET_DIR",
793 "WINBINDD_PRIV_PIPE_DIR",
797 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
799 teardown_env($_) foreach(keys %running_envs);
800 die("Received signal $signame");
807 my $testenv_vars = undef;
813 $option =~ s/^[^:]*//;
816 $option = "client" if $option eq "";
818 if ($envname eq "none") {
820 } elsif (defined(get_running_env($envname))) {
821 $testenv_vars = get_running_env($envname);
822 if (not $target->check_env($testenv_vars)) {
823 print $target->getlog_env($testenv_vars);
824 $testenv_vars = undef;
827 $testenv_vars = $target->setup_env($envname, $prefix);
830 return undef unless defined($testenv_vars);
832 $running_envs{$envname} = $testenv_vars;
834 if ($option eq "local") {
835 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
836 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
837 } elsif ($option eq "client") {
838 SocketWrapper::set_default_iface(11);
839 write_clientconf($conffile, $testenv_vars);
840 $ENV{SMB_CONF_PATH} = $conffile;
842 die("Unknown option[$option] for envname[$envname]");
845 foreach (@exported_envvars) {
846 if (defined($testenv_vars->{$_})) {
847 $ENV{$_} = $testenv_vars->{$_};
853 return $testenv_vars;
856 sub exported_envvars_str($)
858 my ($testenv_vars) = @_;
861 foreach (@exported_envvars) {
862 next unless defined($testenv_vars->{$_});
863 $out .= $_."=".$testenv_vars->{$_}."\n";
872 return "" if ($envname eq "none");
873 return $target->getlog_env(get_running_env($envname));
879 return 1 if ($envname eq "none");
880 return $target->check_env(get_running_env($envname));
886 return if ($envname eq "none");
887 $target->teardown_env(get_running_env($envname));
888 delete $running_envs{$envname};
891 # This 'global' file needs to be empty when we start
892 unlink("$prefix_abs/dns_host_file");
895 my $testenv_name = $ENV{SELFTEST_TESTENV};
896 $testenv_name = $testenv_default unless defined($testenv_name);
898 my $testenv_vars = setup_env($testenv_name);
900 die("Unable to setup environment $testenv_name") unless ($testenv_vars);
902 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
904 my $envvarstr = exported_envvars_str($testenv_vars);
906 my $term = ($ENV{TERMINAL} or "xterm");
907 system("$term -e 'echo -e \"
908 Welcome to the Samba4 Test environment '$testenv_name'
910 This matches the client environment used in make test
911 server is pid `cat \$PIDDIR/samba.pid`
913 Some useful environment variables:
914 TORTURE_OPTIONS=\$TORTURE_OPTIONS
915 SMB_CONF_PATH=\$SMB_CONF_PATH
918 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
919 teardown_env($testenv_name);
925 my $envname = $$_[1];
927 my $envvars = setup_env($envname);
928 if (not defined($envvars)) {
929 Subunit::start_testsuite($name);
930 Subunit::end_testsuite($name, "error",
931 "unable to set up environment $envname");
935 # Generate a file with the individual tests to run, if the
936 # test runner for this test suite supports it.
937 if ($individual_tests and $individual_tests->{$name}) {
939 my ($fh, $listid_file) = tempfile(UNLINK => 0);
940 foreach my $test (@{$individual_tests->{$name}}) {
941 print $fh substr($test, length($name)+1) . "\n";
943 $cmd =~ s/\$LOADLIST/--load-list=$listid_file/g;
945 $cmd =~ s/\s+[^\s]+\s*$//;
946 $cmd .= " " . join(' ', @{$individual_tests->{$name}});
950 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
952 teardown_env($envname) if ($opt_resetup_env);
958 teardown_env($_) foreach (keys %running_envs);
962 # if there were any valgrind failures, show them
963 foreach (<$prefix/valgrind.log*>) {
965 print "VALGRIND FAILURE\n";