2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd qw(abs_path);
140 use Time::HiRes qw(time);
144 my $opt_target = "samba4";
146 my $opt_socket_wrapper = 0;
147 my $opt_socket_wrapper_pcap = undef;
148 my $opt_socket_wrapper_keep_pcap = undef;
150 my @opt_exclude = ();
151 my @opt_include = ();
153 my $opt_image = undef;
156 my $opt_resetup_env = undef;
157 my $opt_bindir = undef;
158 my $opt_load_list = undef;
171 my ($list, $fullname) = @_;
174 if ($fullname =~ /$$_[0]/) {
175 return ($$_[1]) if ($$_[1]);
187 return find_in_list(\@excludes, $name);
196 return unless ($opt_socket_wrapper_pcap);
197 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
200 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
202 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
204 SocketWrapper::setup_pcap($pcap_file);
211 my ($pcap_file, $exitcode) = @_;
213 return unless ($opt_socket_wrapper_pcap);
214 return if ($opt_socket_wrapper_keep_pcap);
215 return unless ($exitcode == 0);
216 return unless defined($pcap_file);
221 # expand strings from %ENV
222 sub expand_environment_strings($)
225 # we use a reverse sort so we do the longer ones first
226 foreach my $k (sort { $b cmp $a } keys %ENV) {
227 $s =~ s/\$$k/$ENV{$k}/g;
232 sub run_testsuite($$$$$)
234 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
235 my $pcap_file = setup_pcap($name);
237 Subunit::start_testsuite($name);
238 Subunit::progress_push();
239 Subunit::report_time(time());
241 Subunit::report_time(time());
242 Subunit::progress_pop();
245 Subunit::progress_pop();
246 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
249 Subunit::end_testsuite($name, "error",
250 sprintf("%s died with signal %d, %s coredump\n", $cmd, ($? & 127), ($? & 128) ? 'with' : 'without'));
254 my $exitcode = $? >> 8;
256 my $envlog = getlog_env($envname);
258 print "envlog: $envlog\n";
261 print "command: $cmd\n";
262 printf "expanded command: %s\n", expand_environment_strings($cmd);
264 if ($exitcode == 0) {
265 Subunit::end_testsuite($name, "success");
267 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
270 cleanup_pcap($pcap_file, $exitcode);
272 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
273 print "PCAP FILE: $pcap_file\n";
276 if ($exitcode != 0) {
277 exit(1) if ($opt_one);
285 print "Samba test runner
286 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
287 Copyright (C) Stefan Metzmacher <metze\@samba.org>
289 Usage: $Script [OPTIONS] TESTNAME-REGEX
292 --help this help page
293 --target=samba[34]|win|kvm Samba version to target
294 --testlist=FILE file to read available tests from
297 --prefix=DIR prefix to run tests in [st]
298 --srcdir=DIR source directory [.]
299 --builddir=DIR output directory [.]
300 --exeext=EXT executable extention []
303 --socket-wrapper-pcap save traffic to pcap directories
304 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
306 --socket-wrapper enable socket wrapper
307 --bindir=PATH path to target binaries
310 --ldap=openldap|fedora-ds back samba onto specified ldap server
313 --image=PATH path to KVM image
316 --quick run quick overall test
317 --one abort when the first test fails
319 --analyse-cmd CMD command to run after each test
324 my $result = GetOptions (
325 'help|h|?' => \$opt_help,
326 'target=s' => \$opt_target,
327 'prefix=s' => \$prefix,
328 'socket-wrapper' => \$opt_socket_wrapper,
329 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
330 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
331 'quick' => \$opt_quick,
333 'exclude=s' => \@opt_exclude,
334 'include=s' => \@opt_include,
335 'srcdir=s' => \$srcdir,
336 'builddir=s' => \$builddir,
337 'exeext=s' => \$exeext,
338 'verbose' => \$opt_verbose,
339 'testenv' => \$opt_testenv,
341 'resetup-environment' => \$opt_resetup_env,
342 'bindir:s' => \$opt_bindir,
343 'image=s' => \$opt_image,
344 'testlist=s' => \@testlists,
345 'load-list=s' => \$opt_load_list,
348 exit(1) if (not $result);
350 ShowHelp() if ($opt_help);
352 # we want unbuffered output
357 # quick hack to disable rpc validation when using valgrind - its way too slow
358 unless (defined($ENV{VALGRIND})) {
359 $ENV{VALIDATE} = "validate";
360 $ENV{MALLOC_CHECK_} = 2;
363 # make all our python scripts unbuffered
364 $ENV{PYTHONUNBUFFERED} = 1;
366 my $bindir = ($opt_bindir or "$builddir/bin");
367 my $bindir_abs = abs_path($bindir);
369 # Backwards compatibility:
370 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
371 if (defined($ENV{FEDORA_DS_ROOT})) {
378 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
381 $torture_maxtime *= 2;
388 die("using an empty prefix isn't allowed") unless $prefix ne "";
390 #Ensure we have the test prefix around
391 mkdir($prefix, 0777) unless -d $prefix;
393 my $prefix_abs = abs_path($prefix);
394 my $srcdir_abs = abs_path($srcdir);
395 my $builddir_abs = abs_path($builddir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX} = $prefix;
401 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS} = $prefix_abs;
403 $ENV{SRCDIR} = $srcdir;
404 $ENV{SRCDIR_ABS} = $srcdir_abs;
405 $ENV{BUILDDIR} = $builddir;
406 $ENV{BUILDDIR_ABS} = $builddir_abs;
407 $ENV{EXEEXT} = $exeext;
409 my $tls_enabled = not $opt_quick;
410 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
411 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
412 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
413 sub prefix_pathvar($$)
415 my ($name, $newpath) = @_;
416 if (defined($ENV{$name})) {
417 $ENV{$name} = "$newpath:$ENV{$name}";
419 $ENV{$name} = $newpath;
422 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
423 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
425 if ($opt_socket_wrapper_keep_pcap) {
426 # Socket wrapper keep pcap implies socket wrapper pcap
427 $opt_socket_wrapper_pcap = 1;
430 if ($opt_socket_wrapper_pcap) {
431 # Socket wrapper pcap implies socket wrapper
432 $opt_socket_wrapper = 1;
435 my $socket_wrapper_dir;
436 if ($opt_socket_wrapper) {
437 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
438 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
441 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
446 my $testenv_default = "none";
448 if ($opt_target eq "samba4") {
449 $testenv_default = "all";
450 require target::Samba4;
451 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
452 } elsif ($opt_target eq "samba3") {
453 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
454 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
456 $testenv_default = "member";
457 require target::Samba3;
458 $target = new Samba3($bindir);
459 } elsif ($opt_target eq "win") {
460 die("Windows tests will not run with socket wrapper enabled.")
461 if ($opt_socket_wrapper);
462 $testenv_default = "dc";
463 require target::Windows;
464 $target = new Windows();
465 } elsif ($opt_target eq "kvm") {
466 die("Kvm tests will not run with socket wrapper enabled.")
467 if ($opt_socket_wrapper);
469 die("No image specified") unless ($opt_image);
470 $target = new Kvm($opt_image, undef);
474 # Start a Virtual Distributed Ethernet Switch
475 # Returns the pid of the switch.
477 sub start_vde_switch($)
481 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
483 open(PID, "$path/vde.pid");
491 # Stop a Virtual Distributed Ethernet Switch
492 sub stop_vde_switch($)
498 sub read_test_regexes($)
502 open(LF, "<$name") or die("unable to read $name: $!");
506 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
507 push (@ret, [$1, $4]);
509 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
510 push (@ret, [$_, undef]);
517 foreach (@opt_exclude) {
518 push (@excludes, read_test_regexes($_));
521 foreach (@opt_include) {
522 push (@includes, read_test_regexes($_));
525 my $interfaces = join(',', ("127.0.0.11/8",
532 my $conffile = "$prefix_abs/client/client.conf";
533 $ENV{SMB_CONF_PATH} = $conffile;
535 sub write_clientconf($$)
537 my ($conffile, $vars) = @_;
539 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
541 if ( -d "$prefix/client/private" ) {
542 unlink <$prefix/client/private/*>;
544 mkdir("$prefix/client/private", 0777);
547 if ( -d "$prefix/client/lockdir" ) {
548 unlink <$prefix/client/lockdir/*>;
550 mkdir("$prefix/client/lockdir", 0777);
553 if ( -d "$prefix_abs/client/ncalrpcdir" ) {
554 unlink <$prefix/client/ncalrpcdir/*>;
556 mkdir("$prefix/client/ncalrpcdir", 0777);
559 open(CF, ">$conffile");
560 print CF "[global]\n";
561 if (defined($ENV{VALGRIND})) {
562 print CF "\ticonv:native = true\n";
564 print CF "\ticonv:native = false\n";
566 print CF "\tnetbios name = client\n";
567 if (defined($vars->{DOMAIN})) {
568 print CF "\tworkgroup = $vars->{DOMAIN}\n";
570 if (defined($vars->{REALM})) {
571 print CF "\trealm = $vars->{REALM}\n";
573 if ($opt_socket_wrapper) {
574 print CF "\tinterfaces = $interfaces\n";
577 private dir = $prefix_abs/client/private
578 lock dir = $prefix_abs/client/lockdir
579 ncalrpc dir = $prefix_abs/client/ncalrpcdir
580 name resolve order = bcast file
581 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
583 notify:inotify = false
585 system:anonymous = true
586 client lanman auth = Yes
588 torture:basedir = $prefix_abs/client
589 #We don't want to pass our self-tests if the PAC code is wrong
590 gensec:require_pac = true
591 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
592 setup directory = ./setup
593 resolv:host file = $prefix_abs/dns_host_file
600 my $testsdir = "$srcdir/selftest";
602 sub should_run_test($)
608 for (my $i=0; $i <= $#tests; $i++) {
609 if ($name =~ /$tests[$i]/i) {
621 open(IN, $filename) or die("Unable to open $filename: $!");
624 if (/-- TEST(-LOADLIST|-IDLIST|) --\n/) {
625 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
626 my $supports_idlist = (defined($1) and $1 eq "-IDLIST");
633 if (should_run_test($name) == 1) {
634 push (@ret, [$name, $env, $cmdline, $supports_loadlist, $supports_idlist]);
640 close(IN) or die("Error creating recipe");
644 if ($#testlists == -1) {
645 die("No testlists specified");
648 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
649 if ($opt_socket_wrapper) {
650 $ENV{SELFTEST_INTERFACES} = $interfaces;
652 $ENV{SELFTEST_INTERFACES} = "";
655 $ENV{SELFTEST_VERBOSE} = "1";
657 $ENV{SELFTEST_VERBOSE} = "";
660 $ENV{SELFTEST_QUICK} = "1";
662 $ENV{SELFTEST_QUICK} = "";
664 $ENV{SELFTEST_TARGET} = $opt_target;
665 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
668 foreach my $fn (@testlists) {
669 foreach (read_testlist($fn)) {
671 next if (@includes and not defined(find_in_list(\@includes, $name)));
672 push (@available, $_);
676 my $restricted = undef;
677 my $restricted_used = {};
679 if ($opt_load_list) {
681 open(LOAD_LIST, "<$opt_load_list") or die("Unable to open $opt_load_list");
682 while (<LOAD_LIST>) {
684 push (@$restricted, $_);
689 my $individual_tests = undef;
690 $individual_tests = {};
692 foreach my $testsuite (@available) {
693 my $name = $$testsuite[0];
694 my $skipreason = skip($name);
695 if (defined($restricted)) {
696 # Find the testsuite for this test
698 foreach my $r (@$restricted) {
700 $individual_tests->{$name} = [];
702 $restricted_used->{$r} = 1;
703 } elsif (substr($r, 0, length($name)+1) eq "$name.") {
704 push(@{$individual_tests->{$name}}, $r);
706 $restricted_used->{$r} = 1;
710 if (defined($skipreason)) {
711 Subunit::skip_testsuite($name, $skipreason);
713 push(@todo, $testsuite);
716 } elsif (defined($skipreason)) {
717 Subunit::skip_testsuite($name, $skipreason);
719 push(@todo, $testsuite);
723 if (defined($restricted)) {
724 foreach (@$restricted) {
725 unless (defined($restricted_used->{$_})) {
726 print "No test or testsuite found matching $_\n";
729 } elsif ($#todo == -1) {
730 print STDERR "No tests to run\n";
734 my $suitestotal = $#todo + 1;
736 Subunit::progress($suitestotal);
737 Subunit::report_time(time());
742 my %running_envs = ();
744 sub get_running_env($)
752 return $running_envs{$envname};
755 my @exported_envvars = (
760 # domain controller stuff
769 "MEMBER_NETBIOSNAME",
770 "MEMBER_NETBIOSALIAS",
772 # rpc proxy controller stuff
774 "RPC_PROXY_SERVER_IP",
775 "RPC_PROXY_NETBIOSNAME",
776 "RPC_PROXY_NETBIOSALIAS",
778 # domain controller stuff for Vampired DC
780 "VAMPIRE_DC_SERVER_IP",
781 "VAMPIRE_DC_NETBIOSNAME",
782 "VAMPIRE_DC_NETBIOSALIAS",
799 "WINBINDD_SOCKET_DIR",
800 "WINBINDD_PRIV_PIPE_DIR",
804 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
806 teardown_env($_) foreach(keys %running_envs);
807 die("Received signal $signame");
814 my $testenv_vars = undef;
820 $option =~ s/^[^:]*//;
823 $option = "client" if $option eq "";
825 if ($envname eq "none") {
827 } elsif (defined(get_running_env($envname))) {
828 $testenv_vars = get_running_env($envname);
829 if (not $target->check_env($testenv_vars)) {
830 print $target->getlog_env($testenv_vars);
831 $testenv_vars = undef;
834 $testenv_vars = $target->setup_env($envname, $prefix);
837 return undef unless defined($testenv_vars);
839 $running_envs{$envname} = $testenv_vars;
841 if ($option eq "local") {
842 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
843 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
844 } elsif ($option eq "client") {
845 SocketWrapper::set_default_iface(11);
846 write_clientconf($conffile, $testenv_vars);
847 $ENV{SMB_CONF_PATH} = $conffile;
849 die("Unknown option[$option] for envname[$envname]");
852 foreach (@exported_envvars) {
853 if (defined($testenv_vars->{$_})) {
854 $ENV{$_} = $testenv_vars->{$_};
860 return $testenv_vars;
863 sub exported_envvars_str($)
865 my ($testenv_vars) = @_;
868 foreach (@exported_envvars) {
869 next unless defined($testenv_vars->{$_});
870 $out .= $_."=".$testenv_vars->{$_}."\n";
879 return "" if ($envname eq "none");
880 return $target->getlog_env(get_running_env($envname));
886 return 1 if ($envname eq "none");
887 return $target->check_env(get_running_env($envname));
893 return if ($envname eq "none");
894 $target->teardown_env(get_running_env($envname));
895 delete $running_envs{$envname};
898 # This 'global' file needs to be empty when we start
899 unlink("$prefix_abs/dns_host_file");
902 my $testenv_name = $ENV{SELFTEST_TESTENV};
903 $testenv_name = $testenv_default unless defined($testenv_name);
905 my $testenv_vars = setup_env($testenv_name);
907 die("Unable to setup environment $testenv_name") unless ($testenv_vars);
909 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
911 my $envvarstr = exported_envvars_str($testenv_vars);
913 my $term = ($ENV{TERMINAL} or "xterm");
914 system("$term -e 'echo -e \"
915 Welcome to the Samba4 Test environment '$testenv_name'
917 This matches the client environment used in make test
918 server is pid `cat \$PIDDIR/samba.pid`
920 Some useful environment variables:
921 TORTURE_OPTIONS=\$TORTURE_OPTIONS
922 SMB_CONF_PATH=\$SMB_CONF_PATH
925 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
926 teardown_env($testenv_name);
932 my $envname = $$_[1];
934 my $envvars = setup_env($envname);
935 if (not defined($envvars)) {
936 Subunit::start_testsuite($name);
937 Subunit::end_testsuite($name, "error",
938 "unable to set up environment $envname");
942 # Generate a file with the individual tests to run, if the
943 # test runner for this test suite supports it.
944 if ($individual_tests and $individual_tests->{$name}) {
946 my ($fh, $listid_file) = tempfile(UNLINK => 0);
947 foreach my $test (@{$individual_tests->{$name}}) {
948 print $fh substr($test, length($name)+1) . "\n";
950 $cmd =~ s/\$LOADLIST/--load-list=$listid_file/g;
952 $cmd =~ s/\s+[^\s]+\s*$//;
953 $cmd .= " " . join(' ', @{$individual_tests->{$name}});
957 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
959 teardown_env($envname) if ($opt_resetup_env);
965 teardown_env($_) foreach (keys %running_envs);
969 # if there were any valgrind failures, show them
970 foreach (<$prefix/valgrind.log*>) {
972 print "VALGRIND FAILURE\n";