2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd qw(abs_path);
133 use Subunit qw(parse_results);
138 my $opt_target = "samba4";
140 my $opt_socket_wrapper = 0;
141 my $opt_socket_wrapper_pcap = undef;
142 my $opt_socket_wrapper_keep_pcap = undef;
144 my @opt_exclude = ();
145 my @opt_include = ();
147 my $opt_image = undef;
150 my $opt_analyse_cmd = undef;
151 my $opt_resetup_env = undef;
152 my $opt_bindir = undef;
153 my $opt_no_lazy_setup = undef;
154 my $opt_load_list = undef;
167 my ($list, $fullname) = @_;
170 if ($fullname =~ /$$_[0]/) {
171 return ($$_[1]) if ($$_[1]);
183 return find_in_list(\@excludes, $name);
192 return unless ($opt_socket_wrapper_pcap);
193 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
196 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
198 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
200 SocketWrapper::setup_pcap($pcap_file);
207 my ($pcap_file, $exitcode) = @_;
209 return unless ($opt_socket_wrapper_pcap);
210 return if ($opt_socket_wrapper_keep_pcap);
211 return unless ($exitcode == 0);
212 return unless defined($pcap_file);
217 # expand strings from %ENV
218 sub expand_environment_strings($)
221 # we use a reverse sort so we do the longer ones first
222 foreach my $k (sort { $b cmp $a } keys %ENV) {
223 $s =~ s/\$$k/$ENV{$k}/g;
228 sub run_testsuite($$$$$)
230 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
231 my $pcap_file = setup_pcap($name);
233 Subunit::start_testsuite($name);
234 Subunit::progress_push();
235 Subunit::report_time(time());
237 open(RESULTS, "$cmd 2>&1|");
239 TESTS_UNEXPECTED_OK => 0,
240 TESTS_EXPECTED_OK => 0,
241 TESTS_UNEXPECTED_FAIL => 0,
242 TESTS_EXPECTED_FAIL => 0,
247 my $msg_ops = new Subunit::Filter("$name\.", []);
249 parse_results($msg_ops, $statistics, *RESULTS);
253 unless (close(RESULTS)) {
255 Subunit::progress_pop();
256 Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!");
264 Subunit::progress_pop();
265 Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without"));
268 my $envlog = getlog_env($envname);
270 print "envlog: $envlog\n";
273 print "command: $cmd\n";
274 printf "expanded command: %s\n", expand_environment_strings($cmd);
276 my $exitcode = $ret >> 8;
278 Subunit::report_time(time());
279 Subunit::progress_pop();
280 if ($exitcode == 0) {
281 Subunit::end_testsuite($name, "success");
283 Subunit::end_testsuite($name, "failure", "Exit code was $exitcode");
286 cleanup_pcap($pcap_file, $exitcode);
288 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
289 print "PCAP FILE: $pcap_file\n";
292 if ($exitcode != 0) {
293 exit(1) if ($opt_one);
301 print "Samba test runner
302 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
303 Copyright (C) Stefan Metzmacher <metze\@samba.org>
305 Usage: $Script [OPTIONS] TESTNAME-REGEX
308 --help this help page
309 --target=samba[34]|win|kvm Samba version to target
310 --testlist=FILE file to read available tests from
313 --prefix=DIR prefix to run tests in [st]
314 --srcdir=DIR source directory [.]
315 --builddir=DIR output directory [.]
316 --exeext=EXT executable extention []
319 --socket-wrapper-pcap save traffic to pcap directories
320 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
322 --socket-wrapper enable socket wrapper
323 --bindir=PATH path to target binaries
326 --ldap=openldap|fedora-ds back samba onto specified ldap server
329 --image=PATH path to KVM image
332 --quick run quick overall test
333 --one abort when the first test fails
335 --analyse-cmd CMD command to run after each test
340 my $result = GetOptions (
341 'help|h|?' => \$opt_help,
342 'target=s' => \$opt_target,
343 'prefix=s' => \$prefix,
344 'socket-wrapper' => \$opt_socket_wrapper,
345 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
346 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
347 'quick' => \$opt_quick,
349 'exclude=s' => \@opt_exclude,
350 'include=s' => \@opt_include,
351 'srcdir=s' => \$srcdir,
352 'builddir=s' => \$builddir,
353 'exeext=s' => \$exeext,
354 'verbose' => \$opt_verbose,
355 'testenv' => \$opt_testenv,
357 'analyse-cmd=s' => \$opt_analyse_cmd,
358 'no-lazy-setup' => \$opt_no_lazy_setup,
359 'resetup-environment' => \$opt_resetup_env,
360 'bindir:s' => \$opt_bindir,
361 'image=s' => \$opt_image,
362 'testlist=s' => \@testlists,
363 'load-list=s' => \$opt_load_list,
366 exit(1) if (not $result);
368 ShowHelp() if ($opt_help);
370 # we want unbuffered output
375 # quick hack to disable rpc validation when using valgrind - its way too slow
376 unless (defined($ENV{VALGRIND})) {
377 $ENV{VALIDATE} = "validate";
378 $ENV{MALLOC_CHECK_} = 2;
381 # make all our python scripts unbuffered
382 $ENV{PYTHONUNBUFFERED} = 1;
384 my $bindir = ($opt_bindir or "$builddir/bin");
385 my $bindir_abs = abs_path($bindir);
387 # Backwards compatibility:
388 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
389 if (defined($ENV{FEDORA_DS_ROOT})) {
396 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
399 $torture_maxtime *= 2;
406 die("using an empty prefix isn't allowed") unless $prefix ne "";
408 #Ensure we have the test prefix around
409 mkdir($prefix, 0777) unless -d $prefix;
411 my $prefix_abs = abs_path($prefix);
412 my $srcdir_abs = abs_path($srcdir);
413 my $builddir_abs = abs_path($builddir);
415 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
416 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
418 $ENV{PREFIX} = $prefix;
419 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
420 $ENV{PREFIX_ABS} = $prefix_abs;
421 $ENV{SRCDIR} = $srcdir;
422 $ENV{SRCDIR_ABS} = $srcdir_abs;
423 $ENV{BUILDDIR} = $builddir;
424 $ENV{BUILDDIR_ABS} = $builddir_abs;
425 $ENV{EXEEXT} = $exeext;
427 my $tls_enabled = not $opt_quick;
428 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
429 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
430 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
431 sub prefix_pathvar($$)
433 my ($name, $newpath) = @_;
434 if (defined($ENV{$name})) {
435 $ENV{$name} = "$newpath:$ENV{$name}";
437 $ENV{$name} = $newpath;
440 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
441 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
443 if ($opt_socket_wrapper_keep_pcap) {
444 # Socket wrapper keep pcap implies socket wrapper pcap
445 $opt_socket_wrapper_pcap = 1;
448 if ($opt_socket_wrapper_pcap) {
449 # Socket wrapper pcap implies socket wrapper
450 $opt_socket_wrapper = 1;
453 my $socket_wrapper_dir;
454 if ($opt_socket_wrapper) {
455 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix_abs/w", $opt_socket_wrapper_pcap);
456 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
459 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
464 my $testenv_default = "none";
466 if ($opt_target eq "samba4") {
467 $testenv_default = "all";
468 require target::Samba4;
469 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
470 } elsif ($opt_target eq "samba3") {
471 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
472 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
474 $testenv_default = "member";
475 require target::Samba3;
476 $target = new Samba3($bindir);
477 } elsif ($opt_target eq "win") {
478 die("Windows tests will not run with socket wrapper enabled.")
479 if ($opt_socket_wrapper);
480 $testenv_default = "dc";
481 require target::Windows;
482 $target = new Windows();
483 } elsif ($opt_target eq "kvm") {
484 die("Kvm tests will not run with socket wrapper enabled.")
485 if ($opt_socket_wrapper);
487 die("No image specified") unless ($opt_image);
488 $target = new Kvm($opt_image, undef);
492 # Start a Virtual Distributed Ethernet Switch
493 # Returns the pid of the switch.
495 sub start_vde_switch($)
499 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
501 open(PID, "$path/vde.pid");
509 # Stop a Virtual Distributed Ethernet Switch
510 sub stop_vde_switch($)
516 sub read_test_regexes($)
520 open(LF, "<$name") or die("unable to read $name: $!");
524 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
525 push (@ret, [$1, $4]);
527 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
528 push (@ret, [$_, undef]);
535 foreach (@opt_exclude) {
536 push (@excludes, read_test_regexes($_));
539 foreach (@opt_include) {
540 push (@includes, read_test_regexes($_));
543 my $interfaces = join(',', ("127.0.0.11/8",
550 my $conffile = "$prefix_abs/client/client.conf";
551 $ENV{SMB_CONF_PATH} = $conffile;
553 sub write_clientconf($$)
555 my ($conffile, $vars) = @_;
557 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
559 if ( -d "$prefix/client/private" ) {
560 unlink <$prefix/client/private/*>;
562 mkdir("$prefix/client/private", 0777);
565 if ( -d "$prefix/client/lockdir" ) {
566 unlink <$prefix/client/lockdir/*>;
568 mkdir("$prefix/client/lockdir", 0777);
571 if ( -d "$prefix_abs/client/ncalrpcdir" ) {
572 unlink <$prefix/client/ncalrpcdir/*>;
574 mkdir("$prefix/client/ncalrpcdir", 0777);
577 open(CF, ">$conffile");
578 print CF "[global]\n";
579 if (defined($ENV{VALGRIND})) {
580 print CF "\ticonv:native = true\n";
582 print CF "\ticonv:native = false\n";
584 print CF "\tnetbios name = client\n";
585 if (defined($vars->{DOMAIN})) {
586 print CF "\tworkgroup = $vars->{DOMAIN}\n";
588 if (defined($vars->{REALM})) {
589 print CF "\trealm = $vars->{REALM}\n";
591 if ($opt_socket_wrapper) {
592 print CF "\tinterfaces = $interfaces\n";
595 private dir = $prefix_abs/client/private
596 lock dir = $prefix_abs/client/lockdir
597 ncalrpc dir = $prefix_abs/client/ncalrpcdir
598 name resolve order = bcast file
599 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
601 notify:inotify = false
603 system:anonymous = true
604 client lanman auth = Yes
606 torture:basedir = $prefix_abs/client
607 #We don't want to pass our self-tests if the PAC code is wrong
608 gensec:require_pac = true
609 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
610 setup directory = ./setup
611 resolv:host file = $prefix_abs/dns_host_file
618 my $testsdir = "$srcdir/selftest";
620 my %required_envs = ();
622 sub should_run_test($)
628 for (my $i=0; $i <= $#tests; $i++) {
629 if ($name =~ /$tests[$i]/i) {
641 open(IN, $filename) or die("Unable to open $filename: $!");
644 if (/-- TEST(-LOADLIST)? --\n/) {
645 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
652 if (should_run_test($name) == 1) {
653 $required_envs{$env} = 1;
654 push (@ret, [$name, $env, $cmdline, $supports_loadlist]);
660 close(IN) or die("Error creating recipe");
664 if ($#testlists == -1) {
665 die("No testlists specified");
668 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
669 if ($opt_socket_wrapper) {
670 $ENV{SELFTEST_INTERFACES} = $interfaces;
672 $ENV{SELFTEST_INTERFACES} = "";
675 $ENV{SELFTEST_VERBOSE} = "1";
677 $ENV{SELFTEST_VERBOSE} = "";
680 $ENV{SELFTEST_QUICK} = "1";
682 $ENV{SELFTEST_QUICK} = "";
684 $ENV{SELFTEST_TARGET} = $opt_target;
685 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
688 foreach my $fn (@testlists) {
689 foreach (read_testlist($fn)) {
691 next if (@includes and not defined(find_in_list(\@includes, $name)));
692 push (@available, $_);
696 my $restricted = undef;
698 if ($opt_load_list) {
700 open(LOAD_LIST, "<$opt_load_list") or die("Unable to open $opt_load_list");
701 while (<LOAD_LIST>) {
703 push (@$restricted, $_);
708 Subunit::progress($#available+1);
709 Subunit::report_time(time());
711 my $individual_tests = undef;
712 $individual_tests = {};
714 foreach my $testsuite (@available) {
715 my $name = $$testsuite[0];
716 my $skipreason = skip($name);
717 if (defined($skipreason)) {
718 Subunit::skip_testsuite($name, $skipreason);
719 } elsif (defined($restricted)) {
720 # Find the testsuite for this test
722 foreach my $r (@$restricted) {
724 $individual_tests->{$name} = [];
727 if ($r =~ /^$name\.(.*)$/) {
728 push(@{$individual_tests->{$name}}, $1);
732 push(@todo, $testsuite) if ($match);
734 push(@todo, $testsuite);
739 print STDERR "No tests to run\n";
743 my $suitestotal = $#todo + 1;
747 my %running_envs = ();
749 sub get_running_env($)
757 return $running_envs{$envname};
760 my @exported_envvars = (
765 # domain controller stuff
774 "MEMBER_NETBIOSNAME",
775 "MEMBER_NETBIOSALIAS",
777 # rpc proxy controller stuff
779 "RPC_PROXY_SERVER_IP",
780 "RPC_PROXY_NETBIOSNAME",
781 "RPC_PROXY_NETBIOSALIAS",
783 # domain controller stuff for Vampired DC
785 "VAMPIRE_DC_SERVER_IP",
786 "VAMPIRE_DC_NETBIOSNAME",
787 "VAMPIRE_DC_NETBIOSALIAS",
804 "WINBINDD_SOCKET_DIR",
805 "WINBINDD_PRIV_PIPE_DIR",
809 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
811 teardown_env($_) foreach(keys %running_envs);
812 die("Received signal $signame");
819 my $testenv_vars = undef;
825 $option =~ s/^[^:]*//;
828 $option = "client" if $option eq "";
830 if ($envname eq "none") {
832 } elsif (defined(get_running_env($envname))) {
833 $testenv_vars = get_running_env($envname);
834 if (not $target->check_env($testenv_vars)) {
835 $testenv_vars = undef;
838 $testenv_vars = $target->setup_env($envname, $prefix);
841 return undef unless defined($testenv_vars);
843 $running_envs{$envname} = $testenv_vars;
845 if ($option eq "local") {
846 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
847 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
848 } elsif ($option eq "client") {
849 SocketWrapper::set_default_iface(11);
850 write_clientconf($conffile, $testenv_vars);
851 $ENV{SMB_CONF_PATH} = $conffile;
853 die("Unknown option[$option] for envname[$envname]");
856 foreach (@exported_envvars) {
857 if (defined($testenv_vars->{$_})) {
858 $ENV{$_} = $testenv_vars->{$_};
864 return $testenv_vars;
867 sub exported_envvars_str($)
869 my ($testenv_vars) = @_;
872 foreach (@exported_envvars) {
873 next unless defined($testenv_vars->{$_});
874 $out .= $_."=".$testenv_vars->{$_}."\n";
883 return "" if ($envname eq "none");
884 return $target->getlog_env(get_running_env($envname));
890 return 1 if ($envname eq "none");
891 return $target->check_env(get_running_env($envname));
897 return if ($envname eq "none");
898 $target->teardown_env(get_running_env($envname));
899 delete $running_envs{$envname};
902 # This 'global' file needs to be empty when we start
903 unlink("$prefix_abs/dns_host_file");
905 if ($opt_no_lazy_setup) {
906 setup_env($_) foreach (keys %required_envs);
910 my $testenv_name = $ENV{SELFTEST_TESTENV};
911 $testenv_name = $testenv_default unless defined($testenv_name);
913 my $testenv_vars = setup_env($testenv_name);
915 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
917 my $envvarstr = exported_envvars_str($testenv_vars);
919 my $term = ($ENV{TERMINAL} or "xterm");
920 system("$term -e 'echo -e \"
921 Welcome to the Samba4 Test environment '$testenv_name'
923 This matches the client environment used in make test
924 server is pid `cat \$PIDDIR/samba.pid`
926 Some useful environment variables:
927 TORTURE_OPTIONS=\$TORTURE_OPTIONS
928 SMB_CONF_PATH=\$SMB_CONF_PATH
931 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
932 teardown_env($testenv_name);
937 $cmd =~ s/([\(\)])/\\$1/g;
939 my $envname = $$_[1];
941 my $envvars = setup_env($envname);
942 if (not defined($envvars)) {
943 Subunit::skip_testsuite($name,
944 "unable to set up environment $envname");
948 # Generate a file with the individual tests to run, if the
949 # test runner for this test suite supports it.
950 if ($$_[3] and $individual_tests and $individual_tests->{$name}) {
951 my ($fh, $listid_file) = tempfile(UNLINK => 0);
952 foreach (@{$individual_tests->{$name}}) {
955 $cmd .= " --load-list=$listid_file";
958 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
961 if (defined($opt_analyse_cmd)) {
962 system("$opt_analyse_cmd \"$name\"");
965 teardown_env($envname) if ($opt_resetup_env);
971 teardown_env($_) foreach (keys %running_envs);
977 # if there were any valgrind failures, show them
978 foreach (<$prefix/valgrind.log*>) {
980 print "VALGRIND FAILURE\n";