2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd qw(abs_path);
140 use Time::HiRes qw(time);
144 my $opt_target = "samba4";
146 my $opt_socket_wrapper = 0;
147 my $opt_socket_wrapper_pcap = undef;
148 my $opt_socket_wrapper_keep_pcap = undef;
150 my @opt_exclude = ();
151 my @opt_include = ();
153 my $opt_image = undef;
156 my $opt_resetup_env = undef;
157 my $opt_bindir = undef;
158 my $opt_load_list = undef;
171 my ($list, $fullname) = @_;
174 if ($fullname =~ /$$_[0]/) {
175 return ($$_[1]) if ($$_[1]);
187 return find_in_list(\@excludes, $name);
196 return unless ($opt_socket_wrapper_pcap);
197 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
200 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
202 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
204 SocketWrapper::setup_pcap($pcap_file);
211 my ($pcap_file, $exitcode) = @_;
213 return unless ($opt_socket_wrapper_pcap);
214 return if ($opt_socket_wrapper_keep_pcap);
215 return unless ($exitcode == 0);
216 return unless defined($pcap_file);
221 # expand strings from %ENV
222 sub expand_environment_strings($)
225 # we use a reverse sort so we do the longer ones first
226 foreach my $k (sort { $b cmp $a } keys %ENV) {
227 $s =~ s/\$$k/$ENV{$k}/g;
232 sub run_testsuite($$$$$)
234 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
235 my $pcap_file = setup_pcap($name);
237 Subunit::start_testsuite($name);
238 Subunit::progress_push();
239 Subunit::report_time(time());
241 Subunit::report_time(time());
242 Subunit::progress_pop();
245 Subunit::progress_pop();
246 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
249 Subunit::end_testsuite($name, "error",
250 sprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127), ($? & 128) ? 'with' : 'without'));
254 my $exitcode = $? >> 8;
256 my $envlog = getlog_env($envname);
258 print "envlog: $envlog\n";
261 print "command: $cmd\n";
262 printf "expanded command: %s\n", expand_environment_strings($cmd);
264 if ($exitcode == 0) {
265 Subunit::end_testsuite($name, "success");
267 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
270 cleanup_pcap($pcap_file, $exitcode);
272 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
273 print "PCAP FILE: $pcap_file\n";
276 if ($exitcode != 0) {
277 exit(1) if ($opt_one);
285 print "Samba test runner
286 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
287 Copyright (C) Stefan Metzmacher <metze\@samba.org>
289 Usage: $Script [OPTIONS] TESTNAME-REGEX
292 --help this help page
293 --target=samba[34]|win|kvm Samba version to target
294 --testlist=FILE file to read available tests from
297 --prefix=DIR prefix to run tests in [st]
298 --srcdir=DIR source directory [.]
299 --builddir=DIR output directory [.]
300 --exeext=EXT executable extention []
303 --socket-wrapper-pcap save traffic to pcap directories
304 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
306 --socket-wrapper enable socket wrapper
307 --bindir=PATH path to target binaries
310 --ldap=openldap|fedora-ds back samba onto specified ldap server
313 --image=PATH path to KVM image
316 --quick run quick overall test
317 --one abort when the first test fails
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'exclude=s' => \@opt_exclude,
334 'include=s' => \@opt_include,
335 'srcdir=s' => \$srcdir,
336 'builddir=s' => \$builddir,
337 'exeext=s' => \$exeext,
338 'verbose' => \$opt_verbose,
339 'testenv' => \$opt_testenv,
341 'resetup-environment' => \$opt_resetup_env,
342 'bindir:s' => \$opt_bindir,
343 'image=s' => \$opt_image,
344 'testlist=s' => \@testlists,
345 'load-list=s' => \$opt_load_list,
348 exit(1) if (not $result);
350 ShowHelp() if ($opt_help);
352 # we want unbuffered output
357 # quick hack to disable rpc validation when using valgrind - its way too slow
358 unless (defined($ENV{VALGRIND})) {
359 $ENV{VALIDATE} = "validate";
360 $ENV{MALLOC_CHECK_} = 2;
363 # make all our python scripts unbuffered
364 $ENV{PYTHONUNBUFFERED} = 1;
366 my $bindir = ($opt_bindir or "$builddir/bin");
367 my $bindir_abs = abs_path($bindir);
369 # Backwards compatibility:
370 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
371 if (defined($ENV{FEDORA_DS_ROOT})) {
378 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
381 $torture_maxtime *= 2;
388 die("using an empty prefix isn't allowed") unless $prefix ne "";
390 #Ensure we have the test prefix around
391 mkdir($prefix, 0777) unless -d $prefix;
393 my $prefix_abs = abs_path($prefix);
394 my $srcdir_abs = abs_path($srcdir);
395 my $builddir_abs = abs_path($builddir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX} = $prefix;
401 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS} = $prefix_abs;
403 $ENV{SRCDIR} = $srcdir;
404 $ENV{SRCDIR_ABS} = $srcdir_abs;
405 $ENV{BUILDDIR} = $builddir;
406 $ENV{BUILDDIR_ABS} = $builddir_abs;
407 $ENV{EXEEXT} = $exeext;
409 my $tls_enabled = not $opt_quick;
410 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
411 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
412 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
413 sub prefix_pathvar($$)
415 my ($name, $newpath) = @_;
416 if (defined($ENV{$name})) {
417 $ENV{$name} = "$newpath:$ENV{$name}";
419 $ENV{$name} = $newpath;
422 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
423 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
425 if ($opt_socket_wrapper_keep_pcap) {
426 # Socket wrapper keep pcap implies socket wrapper pcap
427 $opt_socket_wrapper_pcap = 1;
430 if ($opt_socket_wrapper_pcap) {
431 # Socket wrapper pcap implies socket wrapper
432 $opt_socket_wrapper = 1;
435 my $socket_wrapper_dir;
436 if ($opt_socket_wrapper) {
437 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
438 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
441 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
446 my $testenv_default = "none";
448 if ($opt_target eq "samba4") {
449 $testenv_default = "all";
450 require target::Samba4;
451 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
452 } elsif ($opt_target eq "samba3") {
453 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
454 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
456 $testenv_default = "member";
457 require target::Samba3;
458 $target = new Samba3($bindir);
459 } elsif ($opt_target eq "win") {
460 die("Windows tests will not run with socket wrapper enabled.")
461 if ($opt_socket_wrapper);
462 $testenv_default = "dc";
463 require target::Windows;
464 $target = new Windows();
465 } elsif ($opt_target eq "kvm") {
466 die("Kvm tests will not run with socket wrapper enabled.")
467 if ($opt_socket_wrapper);
469 die("No image specified") unless ($opt_image);
470 $target = new Kvm($opt_image, undef);
474 # Start a Virtual Distributed Ethernet Switch
475 # Returns the pid of the switch.
477 sub start_vde_switch($)
481 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
483 open(PID, "$path/vde.pid");
491 # Stop a Virtual Distributed Ethernet Switch
492 sub stop_vde_switch($)
498 sub read_test_regexes($)
502 open(LF, "<$name") or die("unable to read $name: $!");
506 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
507 push (@ret, [$1, $4]);
509 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
510 push (@ret, [$_, undef]);
517 foreach (@opt_exclude) {
518 push (@excludes, read_test_regexes($_));
521 foreach (@opt_include) {
522 push (@includes, read_test_regexes($_));
525 my $interfaces = join(',', ("127.0.0.11/8",
532 my $clientdir = "$prefix_abs/client";
534 my $conffile = "$clientdir/client.conf";
535 $ENV{SMB_CONF_PATH} = $conffile;
537 sub write_clientconf($$$)
539 my ($conffile, $clientdir, $vars) = @_;
541 mkdir("$clientdir", 0777) unless -d "$clientdir";
543 if ( -d "$clientdir/private" ) {
544 unlink <$clientdir/private/*>;
546 mkdir("$clientdir/private", 0777);
549 if ( -d "$clientdir/lockdir" ) {
550 unlink <$clientdir/lockdir/*>;
552 mkdir("$clientdir/lockdir", 0777);
555 if ( -d "$clientdir/ncalrpcdir" ) {
556 unlink <$clientdir/ncalrpcdir/*>;
558 mkdir("$clientdir/ncalrpcdir", 0777);
561 open(CF, ">$conffile");
562 print CF "[global]\n";
563 if (defined($ENV{VALGRIND})) {
564 print CF "\ticonv:native = true\n";
566 print CF "\ticonv:native = false\n";
568 print CF "\tnetbios name = client\n";
569 if (defined($vars->{DOMAIN})) {
570 print CF "\tworkgroup = $vars->{DOMAIN}\n";
572 if (defined($vars->{REALM})) {
573 print CF "\trealm = $vars->{REALM}\n";
575 if ($opt_socket_wrapper) {
576 print CF "\tinterfaces = $interfaces\n";
579 private dir = $clientdir/private
580 lock dir = $clientdir/lockdir
581 ncalrpc dir = $clientdir/ncalrpcdir
582 name resolve order = bcast file
583 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
585 notify:inotify = false
587 system:anonymous = true
588 client lanman auth = Yes
590 torture:basedir = $clientdir
591 #We don't want to pass our self-tests if the PAC code is wrong
592 gensec:require_pac = true
593 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
594 setup directory = ./setup
595 resolv:host file = $prefix_abs/dns_host_file
602 my $testsdir = "$srcdir/selftest";
604 sub should_run_test($)
610 for (my $i=0; $i <= $#tests; $i++) {
611 if ($name =~ /$tests[$i]/i) {
623 open(IN, $filename) or die("Unable to open $filename: $!");
626 if (/-- TEST(-LOADLIST|-IDLIST|) --\n/) {
627 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
628 my $supports_idlist = (defined($1) and $1 eq "-IDLIST");
635 if (should_run_test($name) == 1) {
636 push (@ret, [$name, $env, $cmdline, $supports_loadlist, $supports_idlist]);
642 close(IN) or die("Error creating recipe");
646 if ($#testlists == -1) {
647 die("No testlists specified");
650 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
651 if ($opt_socket_wrapper) {
652 $ENV{SELFTEST_INTERFACES} = $interfaces;
654 $ENV{SELFTEST_INTERFACES} = "";
657 $ENV{SELFTEST_VERBOSE} = "1";
659 $ENV{SELFTEST_VERBOSE} = "";
662 $ENV{SELFTEST_QUICK} = "1";
664 $ENV{SELFTEST_QUICK} = "";
666 $ENV{SELFTEST_TARGET} = $opt_target;
667 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
670 foreach my $fn (@testlists) {
671 foreach (read_testlist($fn)) {
673 next if (@includes and not defined(find_in_list(\@includes, $name)));
674 push (@available, $_);
678 my $restricted = undef;
679 my $restricted_used = {};
681 if ($opt_load_list) {
683 open(LOAD_LIST, "<$opt_load_list") or die("Unable to open $opt_load_list");
684 while (<LOAD_LIST>) {
686 push (@$restricted, $_);
691 my $individual_tests = undef;
692 $individual_tests = {};
694 foreach my $testsuite (@available) {
695 my $name = $$testsuite[0];
696 my $skipreason = skip($name);
697 if (defined($restricted)) {
698 # Find the testsuite for this test
700 foreach my $r (@$restricted) {
702 $individual_tests->{$name} = [];
704 $restricted_used->{$r} = 1;
705 } elsif (substr($r, 0, length($name)+1) eq "$name.") {
706 push(@{$individual_tests->{$name}}, $r);
708 $restricted_used->{$r} = 1;
712 if (defined($skipreason)) {
713 Subunit::skip_testsuite($name, $skipreason);
715 push(@todo, $testsuite);
718 } elsif (defined($skipreason)) {
719 Subunit::skip_testsuite($name, $skipreason);
721 push(@todo, $testsuite);
725 if (defined($restricted)) {
726 foreach (@$restricted) {
727 unless (defined($restricted_used->{$_})) {
728 print "No test or testsuite found matching $_\n";
731 } elsif ($#todo == -1) {
732 print STDERR "No tests to run\n";
736 my $suitestotal = $#todo + 1;
738 Subunit::progress($suitestotal);
739 Subunit::report_time(time());
744 my %running_envs = ();
746 sub get_running_env($)
754 return $running_envs{$envname};
757 my @exported_envvars = (
762 # domain controller stuff
771 "MEMBER_NETBIOSNAME",
772 "MEMBER_NETBIOSALIAS",
774 # rpc proxy controller stuff
776 "RPC_PROXY_SERVER_IP",
777 "RPC_PROXY_NETBIOSNAME",
778 "RPC_PROXY_NETBIOSALIAS",
780 # domain controller stuff for Vampired DC
782 "VAMPIRE_DC_SERVER_IP",
783 "VAMPIRE_DC_NETBIOSNAME",
784 "VAMPIRE_DC_NETBIOSALIAS",
801 "WINBINDD_SOCKET_DIR",
802 "WINBINDD_PRIV_PIPE_DIR",
806 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
808 teardown_env($_) foreach(keys %running_envs);
809 die("Received signal $signame");
816 my $testenv_vars = undef;
822 $option =~ s/^[^:]*//;
825 $option = "client" if $option eq "";
827 if ($envname eq "none") {
829 } elsif (defined(get_running_env($envname))) {
830 $testenv_vars = get_running_env($envname);
831 if (not $target->check_env($testenv_vars)) {
832 print $target->getlog_env($testenv_vars);
833 $testenv_vars = undef;
836 $testenv_vars = $target->setup_env($envname, $prefix);
839 return undef unless defined($testenv_vars);
841 $running_envs{$envname} = $testenv_vars;
843 if ($option eq "local") {
844 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
845 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
846 } elsif ($option eq "client") {
847 SocketWrapper::set_default_iface(11);
848 write_clientconf($conffile, $clientdir, $testenv_vars);
849 $ENV{SMB_CONF_PATH} = $conffile;
851 die("Unknown option[$option] for envname[$envname]");
854 foreach (@exported_envvars) {
855 if (defined($testenv_vars->{$_})) {
856 $ENV{$_} = $testenv_vars->{$_};
862 return $testenv_vars;
865 sub exported_envvars_str($)
867 my ($testenv_vars) = @_;
870 foreach (@exported_envvars) {
871 next unless defined($testenv_vars->{$_});
872 $out .= $_."=".$testenv_vars->{$_}."\n";
881 return "" if ($envname eq "none");
882 return $target->getlog_env(get_running_env($envname));
888 return 1 if ($envname eq "none");
889 return $target->check_env(get_running_env($envname));
895 return if ($envname eq "none");
896 $target->teardown_env(get_running_env($envname));
897 delete $running_envs{$envname};
900 # This 'global' file needs to be empty when we start
901 unlink("$prefix_abs/dns_host_file");
904 my $testenv_name = $ENV{SELFTEST_TESTENV};
905 $testenv_name = $testenv_default unless defined($testenv_name);
907 my $testenv_vars = setup_env($testenv_name);
909 die("Unable to setup environment $testenv_name") unless ($testenv_vars);
911 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
913 my $envvarstr = exported_envvars_str($testenv_vars);
915 my $term = ($ENV{TERMINAL} or "xterm");
916 system("$term -e 'echo -e \"
917 Welcome to the Samba4 Test environment '$testenv_name'
919 This matches the client environment used in make test
920 server is pid `cat \$PIDDIR/samba.pid`
922 Some useful environment variables:
923 TORTURE_OPTIONS=\$TORTURE_OPTIONS
924 SMB_CONF_PATH=\$SMB_CONF_PATH
927 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
928 teardown_env($testenv_name);
934 my $envname = $$_[1];
936 my $envvars = setup_env($envname);
937 if (not defined($envvars)) {
938 Subunit::start_testsuite($name);
939 Subunit::end_testsuite($name, "error",
940 "unable to set up environment $envname");
944 # Generate a file with the individual tests to run, if the
945 # test runner for this test suite supports it.
946 if ($individual_tests and $individual_tests->{$name}) {
948 my ($fh, $listid_file) = tempfile(UNLINK => 0);
949 foreach my $test (@{$individual_tests->{$name}}) {
950 print $fh substr($test, length($name)+1) . "\n";
952 $cmd =~ s/\$LOADLIST/--load-list=$listid_file/g;
954 $cmd =~ s/\s+[^\s]+\s*$//;
955 $cmd .= " " . join(' ', @{$individual_tests->{$name}});
959 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
961 teardown_env($envname) if ($opt_resetup_env);
967 teardown_env($_) foreach (keys %running_envs);
971 # if there were any valgrind failures, show them
972 foreach (<$prefix/valgrind.log*>) {
974 print "VALGRIND FAILURE\n";