2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin qw($RealBin $Script);
130 use Cwd qw(abs_path);
132 use Subunit qw(parse_results);
136 my $opt_target = "samba4";
138 my $opt_socket_wrapper = 0;
139 my $opt_socket_wrapper_pcap = undef;
140 my $opt_socket_wrapper_keep_pcap = undef;
142 my @opt_exclude = ();
143 my @opt_include = ();
145 my $opt_image = undef;
148 my $opt_analyse_cmd = undef;
149 my $opt_resetup_env = undef;
150 my $opt_bindir = undef;
151 my $opt_no_lazy_setup = undef;
164 my ($list, $fullname) = @_;
167 if ($fullname =~ /$$_[0]/) {
168 return ($$_[1]) if ($$_[1]);
169 return "NO REASON SPECIFIED";
180 return find_in_list(\@excludes, $name);
189 return unless ($opt_socket_wrapper_pcap);
190 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
193 $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
195 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
197 SocketWrapper::setup_pcap($pcap_file);
204 my ($pcap_file, $exitcode) = @_;
206 return unless ($opt_socket_wrapper_pcap);
207 return if ($opt_socket_wrapper_keep_pcap);
208 return unless ($exitcode == 0);
209 return unless defined($pcap_file);
214 sub run_testsuite($$$$$)
216 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
217 my $pcap_file = setup_pcap($name);
219 Subunit::report_time(time());
220 Subunit::start_test($name);
222 my $ret = system("$cmd | $RealBin/filter-subunit.pl --prefix \"$name.\" 2>&1");
224 Subunit::end_test($name, "error", "Unable to run $cmd: $!");
227 my $envlog = getlog_env($envname);
229 print "envlog: $envlog\n";
232 print "command: $cmd\n";
234 my $exitcode = $ret >> 8;
236 Subunit::report_time(time());
237 my $reason = "Exit code was $exitcode";
239 if ($exitcode == 0) {
244 Subunit::end_test($name, $result, $reason);
246 cleanup_pcap($pcap_file, $exitcode);
248 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
249 print "PCAP FILE: $pcap_file\n";
252 if ($exitcode != 0) {
253 exit(1) if ($opt_one);
261 print "Samba test runner
262 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
263 Copyright (C) Stefan Metzmacher <metze\@samba.org>
265 Usage: $Script [OPTIONS] TESTNAME-REGEX
268 --help this help page
269 --target=samba[34]|win|kvm Samba version to target
270 --testlist=FILE file to read available tests from
273 --prefix=DIR prefix to run tests in [st]
274 --srcdir=DIR source directory [.]
275 --builddir=DIR output directory [.]
276 --exeext=EXT executable extention []
279 --socket-wrapper-pcap save traffic to pcap directories
280 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
282 --socket-wrapper enable socket wrapper
283 --bindir=PATH path to target binaries
286 --ldap=openldap|fedora-ds back samba onto specified ldap server
289 --image=PATH path to KVM image
292 --quick run quick overall test
293 --one abort when the first test fails
295 --analyse-cmd CMD command to run after each test
300 my $result = GetOptions (
301 'help|h|?' => \$opt_help,
302 'target=s' => \$opt_target,
303 'prefix=s' => \$prefix,
304 'socket-wrapper' => \$opt_socket_wrapper,
305 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
306 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
307 'quick' => \$opt_quick,
309 'exclude=s' => \@opt_exclude,
310 'include=s' => \@opt_include,
311 'srcdir=s' => \$srcdir,
312 'builddir=s' => \$builddir,
313 'exeext=s' => \$exeext,
314 'verbose' => \$opt_verbose,
315 'testenv' => \$opt_testenv,
317 'analyse-cmd=s' => \$opt_analyse_cmd,
318 'no-lazy-setup' => \$opt_no_lazy_setup,
319 'resetup-environment' => \$opt_resetup_env,
320 'bindir:s' => \$opt_bindir,
321 'image=s' => \$opt_image,
322 'testlist=s' => \@testlists
325 exit(1) if (not $result);
327 ShowHelp() if ($opt_help);
331 # quick hack to disable rpc validation when using valgrind - its way too slow
332 unless (defined($ENV{VALGRIND})) {
333 $ENV{VALIDATE} = "validate";
334 $ENV{MALLOC_CHECK_} = 2;
337 my $bindir = ($opt_bindir or "$builddir/bin");
338 my $bindir_abs = abs_path($bindir);
340 # Backwards compatibility:
341 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
342 if (defined($ENV{FEDORA_DS_ROOT})) {
349 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
352 $torture_maxtime *= 2;
359 die("using an empty prefix isn't allowed") unless $prefix ne "";
361 #Ensure we have the test prefix around
362 mkdir($prefix, 0777) unless -d $prefix;
364 my $prefix_abs = abs_path($prefix);
365 my $srcdir_abs = abs_path($srcdir);
366 my $builddir_abs = abs_path($builddir);
368 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
369 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
371 $ENV{PREFIX} = $prefix;
372 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
373 $ENV{PREFIX_ABS} = $prefix_abs;
374 $ENV{SRCDIR} = $srcdir;
375 $ENV{SRCDIR_ABS} = $srcdir_abs;
376 $ENV{BUILDDIR} = $builddir;
377 $ENV{BUILDDIR_ABS} = $builddir_abs;
378 $ENV{EXEEXT} = $exeext;
380 my $tls_enabled = not $opt_quick;
381 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
382 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
383 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
384 sub prefix_pathvar($$)
386 my ($name, $newpath) = @_;
387 if (defined($ENV{$name})) {
388 $ENV{$name} = "$newpath:$ENV{$name}";
390 $ENV{$name} = $newpath;
393 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
394 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
396 if ($opt_socket_wrapper_keep_pcap) {
397 # Socket wrapper keep pcap implies socket wrapper pcap
398 $opt_socket_wrapper_pcap = 1;
401 if ($opt_socket_wrapper_pcap) {
402 # Socket wrapper pcap implies socket wrapper
403 $opt_socket_wrapper = 1;
406 my $socket_wrapper_dir;
407 if ($opt_socket_wrapper) {
408 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
409 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
412 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
417 my $testenv_default = "none";
419 if ($opt_target eq "samba4") {
420 $testenv_default = "member";
421 require target::Samba4;
422 $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
423 } elsif ($opt_target eq "samba3") {
424 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
425 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
427 $testenv_default = "member";
428 require target::Samba3;
429 $target = new Samba3($bindir);
430 } elsif ($opt_target eq "win") {
431 die("Windows tests will not run with socket wrapper enabled.")
432 if ($opt_socket_wrapper);
433 $testenv_default = "dc";
434 require target::Windows;
435 $target = new Windows();
436 } elsif ($opt_target eq "kvm") {
437 die("Kvm tests will not run with socket wrapper enabled.")
438 if ($opt_socket_wrapper);
440 die("No image specified") unless ($opt_image);
441 $target = new Kvm($opt_image, undef);
445 # Start a Virtual Distributed Ethernet Switch
446 # Returns the pid of the switch.
448 sub start_vde_switch($)
452 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
454 open(PID, "$path/vde.pid");
462 # Stop a Virtual Distributed Ethernet Switch
463 sub stop_vde_switch($)
469 sub read_test_regexes($)
473 open(LF, "<$name") or die("unable to read $name: $!");
477 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
478 push (@ret, [$1, $4]);
480 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
481 push (@ret, [$_, undef]);
488 foreach (@opt_exclude) {
489 push (@excludes, read_test_regexes($_));
492 foreach (@opt_include) {
493 push (@includes, read_test_regexes($_));
496 my $interfaces = join(',', ("127.0.0.6/8",
503 my $conffile = "$prefix_abs/client/client.conf";
504 $ENV{SMB_CONF_PATH} = $conffile;
506 sub write_clientconf($$)
508 my ($conffile, $vars) = @_;
510 mkdir("$prefix/client", 0777) unless -d "$prefix/client";
512 if ( -d "$prefix/client/private" ) {
513 unlink <$prefix/client/private/*>;
515 mkdir("$prefix/client/private", 0777);
518 if ( -d "$prefix/client/lock" ) {
519 unlink <$prefix/client/lockdir/*>;
521 mkdir("$prefix/client/lockdir", 0777);
524 open(CF, ">$conffile");
525 print CF "[global]\n";
526 if (defined($ENV{VALGRIND})) {
527 print CF "\ticonv:native = true\n";
529 print CF "\ticonv:native = false\n";
531 print CF "\tnetbios name = client\n";
532 if (defined($vars->{DOMAIN})) {
533 print CF "\tworkgroup = $vars->{DOMAIN}\n";
535 if (defined($vars->{REALM})) {
536 print CF "\trealm = $vars->{REALM}\n";
538 if ($opt_socket_wrapper) {
539 print CF "\tinterfaces = $interfaces\n";
542 private dir = $prefix_abs/client/private
543 lock dir = $prefix_abs/client/lockdir
544 name resolve order = bcast
545 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
547 notify:inotify = false
549 system:anonymous = true
550 client lanman auth = Yes
551 torture:basedir = $prefix_abs/client
552 #We don't want to pass our self-tests if the PAC code is wrong
553 gensec:require_pac = true
554 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
561 my $testsdir = "$srcdir/selftest";
563 my %required_envs = ();
570 open(IN, $filename) or die("Unable to open $filename: $!");
573 if ($_ eq "-- TEST --\n") {
580 if (not defined($tests) or $name =~ /$tests/) {
581 $required_envs{$env} = 1;
582 push (@ret, [$name, $env, $cmdline]);
588 close(IN) or die("Error creating recipe");
592 if ($#testlists == -1) {
593 die("No testlists specified");
596 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
597 if ($opt_socket_wrapper) {
598 $ENV{SELFTEST_INTERFACES} = $interfaces;
600 $ENV{SELFTEST_INTERFACES} = "";
603 $ENV{SELFTEST_VERBOSE} = "1";
605 $ENV{SELFTEST_VERBOSE} = "";
608 $ENV{SELFTEST_QUICK} = "1";
610 $ENV{SELFTEST_QUICK} = "";
612 $ENV{SELFTEST_TARGET} = $opt_target;
613 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
616 foreach my $fn (@testlists) {
617 foreach (read_testlist($fn)) {
619 next if (@includes and not find_in_list(\@includes, $name));
620 push (@available, $_);
624 Subunit::report_time(time());
626 foreach (@available) {
628 my $skipreason = skip($name);
630 Subunit::end_test($name, "skip", $skipreason);
637 print STDERR "No tests to run\n";
641 my $suitestotal = $#todo + 1;
645 my %running_envs = ();
647 sub get_running_env($)
655 return $running_envs{$envname};
658 my @exported_envvars = (
663 # domain controller stuff
683 "WINBINDD_SOCKET_DIR",
684 "WINBINDD_PRIV_PIPE_DIR"
687 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
689 teardown_env($_) foreach(keys %running_envs);
690 die("Received signal $signame");
697 my $testenv_vars = undef;
703 $option =~ s/^[^:]*//;
706 $option = "client" if $option eq "";
708 if ($envname eq "none") {
710 } elsif (defined(get_running_env($envname))) {
711 $testenv_vars = get_running_env($envname);
712 if (not $target->check_env($testenv_vars)) {
713 $testenv_vars = undef;
716 $testenv_vars = $target->setup_env($envname, $prefix);
719 return undef unless defined($testenv_vars);
721 $running_envs{$envname} = $testenv_vars;
723 if ($option eq "local") {
724 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
725 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
726 } elsif ($option eq "client") {
727 SocketWrapper::set_default_iface(6);
728 write_clientconf($conffile, $testenv_vars);
729 $ENV{SMB_CONF_PATH} = $conffile;
731 die("Unknown option[$option] for envname[$envname]");
734 foreach (@exported_envvars) {
735 if (defined($testenv_vars->{$_})) {
736 $ENV{$_} = $testenv_vars->{$_};
742 return $testenv_vars;
745 sub exported_envvars_str($)
747 my ($testenv_vars) = @_;
750 foreach (@exported_envvars) {
751 next unless defined($testenv_vars->{$_});
752 $out .= $_."=".$testenv_vars->{$_}."\n";
761 return "" if ($envname eq "none");
762 return $target->getlog_env(get_running_env($envname));
768 return 1 if ($envname eq "none");
769 return $target->check_env(get_running_env($envname));
775 return if ($envname eq "none");
776 $target->teardown_env(get_running_env($envname));
777 delete $running_envs{$envname};
780 if ($opt_no_lazy_setup) {
781 setup_env($_) foreach (keys %required_envs);
785 my $testenv_name = $ENV{SELFTEST_TESTENV};
786 $testenv_name = $testenv_default unless defined($testenv_name);
788 my $testenv_vars = setup_env($testenv_name);
790 $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
792 my $envvarstr = exported_envvars_str($testenv_vars);
794 my $term = ($ENV{TERM} or "xterm");
795 system("$term -e 'echo -e \"
796 Welcome to the Samba4 Test environment '$testenv_name'
798 This matches the client environment used in make test
799 server is pid `cat \$PIDDIR/samba.pid`
801 Some useful environment variables:
802 TORTURE_OPTIONS=\$TORTURE_OPTIONS
803 SMB_CONF_PATH=\$SMB_CONF_PATH
806 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
807 teardown_env($testenv_name);
812 $cmd =~ s/([\(\)])/\\$1/g;
814 my $envname = $$_[1];
816 my $envvars = setup_env($envname);
817 if (not defined($envvars)) {
818 Subunit::end_test($name, "skip",
819 "unable to set up environment $envname");
823 run_testsuite($envname, $name, $cmd, $i, $suitestotal);
825 if (defined($opt_analyse_cmd)) {
826 system("$opt_analyse_cmd \"$name\"");
829 teardown_env($envname) if ($opt_resetup_env);
835 teardown_env($_) foreach (keys %running_envs);
841 # if there were any valgrind failures, show them
842 foreach (<$prefix/valgrind.log*>) {
844 system("grep DWARF2.CFI.reader $_ > /dev/null");
846 print "VALGRIND FAILURE\n";