2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
137 my $opt_target = "samba4";
139 my $opt_socket_wrapper = 0;
140 my $opt_socket_wrapper_pcap = undef;
141 my $opt_socket_wrapper_keep_pcap = undef;
143 my @opt_exclude = ();
144 my @opt_include = ();
146 my $opt_image = undef;
149 my $opt_analyse_cmd = undef;
150 my $opt_resetup_env = undef;
151 my $opt_bindir = undef;
152 my $opt_no_lazy_setup = undef;
165 my ($list, $fullname) = @_;
168 if ($fullname =~ /$$_[0]/) {
169 return ($$_[1]) if ($$_[1]);
181 return find_in_list(\@excludes, $name);
190 return unless ($opt_socket_wrapper_pcap);
191 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
194 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
196 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
198 SocketWrapper::setup_pcap($pcap_file);
205 my ($pcap_file, $exitcode) = @_;
207 return unless ($opt_socket_wrapper_pcap);
208 return if ($opt_socket_wrapper_keep_pcap);
209 return unless ($exitcode == 0);
210 return unless defined($pcap_file);
215 # expand strings from %ENV
216 sub expand_environment_strings($)
219 # we use a reverse sort so we do the longer ones first
220 foreach my $k (sort { $b cmp $a } keys %ENV) {
221 $s =~ s/\$$k/$ENV{$k}/g;
226 sub run_testsuite($$$$$)
228 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
229 my $pcap_file = setup_pcap($name);
231 Subunit::start_testsuite($name);
232 Subunit::progress_push();
233 Subunit::report_time(time());
235 open(RESULTS, "$cmd 2>&1|");
237 TESTS_UNEXPECTED_OK => 0,
238 TESTS_EXPECTED_OK => 0,
239 TESTS_UNEXPECTED_FAIL => 0,
240 TESTS_EXPECTED_FAIL => 0,
245 my $msg_ops = new Subunit::Filter("$name\.", []);
247 parse_results($msg_ops, $statistics, *RESULTS);
251 unless (close(RESULTS)) {
253 Subunit::progress_pop();
254 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
262 Subunit::progress_pop();
263 Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
266 my $envlog = getlog_env($envname);
268 print "envlog: $envlog\n";
271 print "command: $cmd\n";
272 printf "expanded command: %s\n", expand_environment_strings($cmd);
274 my $exitcode = $ret >> 8;
276 Subunit::report_time(time());
277 Subunit::progress_pop();
278 if ($exitcode == 0) {
279 Subunit::end_testsuite($name, "success");
281 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
284 cleanup_pcap($pcap_file, $exitcode);
286 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
287 print "PCAP FILE: $pcap_file\n";
290 if ($exitcode != 0) {
291 exit(1) if ($opt_one);
299 print "Samba test runner
300 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
301 Copyright (C) Stefan Metzmacher <metze\@samba.org>
303 Usage: $Script [OPTIONS] TESTNAME-REGEX
306 --help this help page
307 --target=samba[34]|win|kvm Samba version to target
308 --testlist=FILE file to read available tests from
311 --prefix=DIR prefix to run tests in [st]
312 --srcdir=DIR source directory [.]
313 --builddir=DIR output directory [.]
314 --exeext=EXT executable extention []
317 --socket-wrapper-pcap save traffic to pcap directories
318 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
320 --socket-wrapper enable socket wrapper
321 --bindir=PATH path to target binaries
324 --ldap=openldap|fedora-ds back samba onto specified ldap server
327 --image=PATH path to KVM image
330 --quick run quick overall test
331 --one abort when the first test fails
333 --analyse-cmd CMD command to run after each test
338 my $result = GetOptions (
339 'help|h|?' => \$opt_help,
340 'target=s' => \$opt_target,
341 'prefix=s' => \$prefix,
342 'socket-wrapper' => \$opt_socket_wrapper,
343 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
344 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
345 'quick' => \$opt_quick,
347 'exclude=s' => \@opt_exclude,
348 'include=s' => \@opt_include,
349 'srcdir=s' => \$srcdir,
350 'builddir=s' => \$builddir,
351 'exeext=s' => \$exeext,
352 'verbose' => \$opt_verbose,
353 'testenv' => \$opt_testenv,
355 'analyse-cmd=s' => \$opt_analyse_cmd,
356 'no-lazy-setup' => \$opt_no_lazy_setup,
357 'resetup-environment' => \$opt_resetup_env,
358 'bindir:s' => \$opt_bindir,
359 'image=s' => \$opt_image,
360 'testlist=s' => \@testlists
363 exit(1) if (not $result);
365 ShowHelp() if ($opt_help);
367 # we want unbuffered output
372 # quick hack to disable rpc validation when using valgrind - its way too slow
373 unless (defined($ENV{VALGRIND})) {
374 $ENV{VALIDATE} = "validate";
375 $ENV{MALLOC_CHECK_} = 2;
378 # make all our python scripts unbuffered
379 $ENV{PYTHONUNBUFFERED} = 1;
381 my $bindir = ($opt_bindir or "$builddir/bin");
382 my $bindir_abs = abs_path($bindir);
384 # Backwards compatibility:
385 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
386 if (defined($ENV{FEDORA_DS_ROOT})) {
393 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
396 $torture_maxtime *= 2;
403 die("using an empty prefix isn't allowed") unless $prefix ne "";
405 #Ensure we have the test prefix around
406 mkdir($prefix, 0777) unless -d $prefix;
408 my $prefix_abs = abs_path($prefix);
409 my $srcdir_abs = abs_path($srcdir);
410 my $builddir_abs = abs_path($builddir);
412 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
413 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
415 $ENV{PREFIX} = $prefix;
416 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
417 $ENV{PREFIX_ABS} = $prefix_abs;
418 $ENV{SRCDIR} = $srcdir;
419 $ENV{SRCDIR_ABS} = $srcdir_abs;
420 $ENV{BUILDDIR} = $builddir;
421 $ENV{BUILDDIR_ABS} = $builddir_abs;
422 $ENV{EXEEXT} = $exeext;
424 my $tls_enabled = not $opt_quick;
425 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
426 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
427 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
428 sub prefix_pathvar($$)
430 my ($name, $newpath) = @_;
431 if (defined($ENV{$name})) {
432 $ENV{$name} = "$newpath:$ENV{$name}";
434 $ENV{$name} = $newpath;
437 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
438 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
440 if ($opt_socket_wrapper_keep_pcap) {
441 # Socket wrapper keep pcap implies socket wrapper pcap
442 $opt_socket_wrapper_pcap = 1;
445 if ($opt_socket_wrapper_pcap) {
446 # Socket wrapper pcap implies socket wrapper
447 $opt_socket_wrapper = 1;
450 my $socket_wrapper_dir;
451 if ($opt_socket_wrapper) {
452 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
453 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
456 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
461 my $testenv_default = "none";
463 if ($opt_target eq "samba4") {
464 $testenv_default = "all";
465 require target::Samba4;
466 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
467 } elsif ($opt_target eq "samba3") {
468 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
469 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
471 $testenv_default = "member";
472 require target::Samba3;
473 $target = new Samba3($bindir);
474 } elsif ($opt_target eq "win") {
475 die("Windows tests will not run with socket wrapper enabled.")
476 if ($opt_socket_wrapper);
477 $testenv_default = "dc";
478 require target::Windows;
479 $target = new Windows();
480 } elsif ($opt_target eq "kvm") {
481 die("Kvm tests will not run with socket wrapper enabled.")
482 if ($opt_socket_wrapper);
484 die("No image specified") unless ($opt_image);
485 $target = new Kvm($opt_image, undef);
489 # Start a Virtual Distributed Ethernet Switch
490 # Returns the pid of the switch.
492 sub start_vde_switch($)
496 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
498 open(PID, "$path/vde.pid");
506 # Stop a Virtual Distributed Ethernet Switch
507 sub stop_vde_switch($)
513 sub read_test_regexes($)
517 open(LF, "<$name") or die("unable to read $name: $!");
521 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
522 push (@ret, [$1, $4]);
524 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
525 push (@ret, [$_, undef]);
532 foreach (@opt_exclude) {
533 push (@excludes, read_test_regexes($_));
536 foreach (@opt_include) {
537 push (@includes, read_test_regexes($_));
540 my $interfaces = join(',', ("127.0.0.6/8",
547 my $conffile = "$prefix_abs/client/client.conf";
548 $ENV{SMB_CONF_PATH} = $conffile;
550 sub write_clientconf($$)
552 my ($conffile, $vars) = @_;
554 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
556 if ( -d "$prefix/client/private" ) {
557 unlink <$prefix/client/private/*>;
559 mkdir("$prefix/client/private", 0777);
562 if ( -d "$prefix/client/lockdir" ) {
563 unlink <$prefix/client/lockdir/*>;
565 mkdir("$prefix/client/lockdir", 0777);
568 if ( -d "$prefix_abs/client/ncalrpcdir" ) {
569 unlink <$prefix/client/ncalrpcdir/*>;
571 mkdir("$prefix/client/ncalrpcdir", 0777);
574 open(CF, ">$conffile");
575 print CF "[global]\n";
576 if (defined($ENV{VALGRIND})) {
577 print CF "\ticonv:native = true\n";
579 print CF "\ticonv:native = false\n";
581 print CF "\tnetbios name = client\n";
582 if (defined($vars->{DOMAIN})) {
583 print CF "\tworkgroup = $vars->{DOMAIN}\n";
585 if (defined($vars->{REALM})) {
586 print CF "\trealm = $vars->{REALM}\n";
588 if ($opt_socket_wrapper) {
589 print CF "\tinterfaces = $interfaces\n";
592 private dir = $prefix_abs/client/private
593 lock dir = $prefix_abs/client/lockdir
594 ncalrpc dir = $prefix_abs/client/ncalrpcdir
595 name resolve order = bcast file
596 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
598 notify:inotify = false
600 system:anonymous = true
601 client lanman auth = Yes
602 torture:basedir = $prefix_abs/client
603 #We don't want to pass our self-tests if the PAC code is wrong
604 gensec:require_pac = true
605 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
606 setup directory = ./setup
607 resolv:host file = $prefix_abs/dns_host_file
614 my $testsdir = "$srcdir/selftest";
616 my %required_envs = ();
618 sub should_run_test($)
624 for (my $i=0; $i <= $#tests; $i++) {
625 if ($name =~ /$tests[$i]/i) {
637 open(IN, $filename) or die("Unable to open $filename: $!");
640 if ($_ eq "-- TEST --\n") {
647 if (should_run_test($name) == 1) {
648 $required_envs{$env} = 1;
649 push (@ret, [$name, $env, $cmdline]);
655 close(IN) or die("Error creating recipe");
659 if ($#testlists == -1) {
660 die("No testlists specified");
663 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
664 if ($opt_socket_wrapper) {
665 $ENV{SELFTEST_INTERFACES} = $interfaces;
667 $ENV{SELFTEST_INTERFACES} = "";
670 $ENV{SELFTEST_VERBOSE} = "1";
672 $ENV{SELFTEST_VERBOSE} = "";
675 $ENV{SELFTEST_QUICK} = "1";
677 $ENV{SELFTEST_QUICK} = "";
679 $ENV{SELFTEST_TARGET} = $opt_target;
680 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
683 foreach my $fn (@testlists) {
684 foreach (read_testlist($fn)) {
686 next if (@includes and not defined(find_in_list(\@includes, $name)));
687 push (@available, $_);
691 Subunit::progress($#available+1);
692 Subunit::report_time(time());
694 foreach (@available) {
696 my $skipreason = skip($name);
697 if (defined($skipreason)) {
698 Subunit::skip_testsuite($name, $skipreason);
705 print STDERR "No tests to run\n";
709 my $suitestotal = $#todo + 1;
713 my %running_envs = ();
715 sub get_running_env($)
723 return $running_envs{$envname};
726 my @exported_envvars = (
731 # domain controller stuff
737 # domain controller stuff
740 "MEMBER_NETBIOSNAME",
741 "MEMBER_NETBIOSALIAS",
743 # domain controller stuff
745 "RPC_PROXY_SERVER_IP",
746 "RPC_PROXY_NETBIOSNAME",
747 "RPC_PROXY_NETBIOSALIAS",
763 "WINBINDD_SOCKET_DIR",
764 "WINBINDD_PRIV_PIPE_DIR"
767 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
769 teardown_env($_) foreach(keys %running_envs);
770 die("Received signal $signame");
777 my $testenv_vars = undef;
783 $option =~ s/^[^:]*//;
786 $option = "client" if $option eq "";
788 if ($envname eq "none") {
790 } elsif (defined(get_running_env($envname))) {
791 $testenv_vars = get_running_env($envname);
792 if (not $target->check_env($testenv_vars)) {
793 $testenv_vars = undef;
796 $testenv_vars = $target->setup_env($envname, $prefix);
799 return undef unless defined($testenv_vars);
801 $running_envs{$envname} = $testenv_vars;
803 if ($option eq "local") {
804 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
805 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
806 } elsif ($option eq "client") {
807 SocketWrapper::set_default_iface(6);
808 write_clientconf($conffile, $testenv_vars);
809 $ENV{SMB_CONF_PATH} = $conffile;
811 die("Unknown option[$option] for envname[$envname]");
814 foreach (@exported_envvars) {
815 if (defined($testenv_vars->{$_})) {
816 $ENV{$_} = $testenv_vars->{$_};
822 return $testenv_vars;
825 sub exported_envvars_str($)
827 my ($testenv_vars) = @_;
830 foreach (@exported_envvars) {
831 next unless defined($testenv_vars->{$_});
832 $out .= $_."=".$testenv_vars->{$_}."\n";
841 return "" if ($envname eq "none");
842 return $target->getlog_env(get_running_env($envname));
848 return 1 if ($envname eq "none");
849 return $target->check_env(get_running_env($envname));
855 return if ($envname eq "none");
856 $target->teardown_env(get_running_env($envname));
857 delete $running_envs{$envname};
860 # This 'global' file needs to be empty when we start
861 unlink("$prefix_abs/dns_host_file");
863 if ($opt_no_lazy_setup) {
864 setup_env($_) foreach (keys %required_envs);
868 my $testenv_name = $ENV{SELFTEST_TESTENV};
869 $testenv_name = $testenv_default unless defined($testenv_name);
871 my $testenv_vars = setup_env($testenv_name);
873 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
875 my $envvarstr = exported_envvars_str($testenv_vars);
877 my $term = ($ENV{TERMINAL} or "xterm");
878 system("$term -e 'echo -e \"
879 Welcome to the Samba4 Test environment '$testenv_name'
881 This matches the client environment used in make test
882 server is pid `cat \$PIDDIR/samba.pid`
884 Some useful environment variables:
885 TORTURE_OPTIONS=\$TORTURE_OPTIONS
886 SMB_CONF_PATH=\$SMB_CONF_PATH
889 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
890 teardown_env($testenv_name);
895 $cmd =~ s/([\(\)])/\\$1/g;
897 my $envname = $$_[1];
899 my $envvars = setup_env($envname);
900 if (not defined($envvars)) {
901 Subunit::skip_testsuite($name,
902 "unable to set up environment $envname");
906 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
908 if (defined($opt_analyse_cmd)) {
909 system("$opt_analyse_cmd \"$name\"");
912 teardown_env($envname) if ($opt_resetup_env);
918 teardown_env($_) foreach (keys %running_envs);
924 # if there were any valgrind failures, show them
925 foreach (<$prefix/valgrind.log*>) {
927 print "VALGRIND FAILURE\n";