1dbb494edbc6e845617607c0242e71502cc3ba42
[kai/samba.git] / source4 / selftest / selftest.pl
1 #!/usr/bin/perl
2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
5
6 =pod
7
8 =head1 NAME
9
10 selftest - Samba test runner
11
12 =head1 SYNOPSIS
13
14 selftest --help
15
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
17
18 =head1 DESCRIPTION
19
20 A simple test runner. TESTS is a regular expression with tests to run.
21
22 =head1 OPTIONS
23
24 =over 4
25
26 =item I<--help>
27
28 Show list of available options.
29
30 =item I<--srcdir=DIR>
31
32 Source directory.
33
34 =item I<--builddir=DIR>
35
36 Build directory.
37
38 =item I<--prefix=DIR>
39
40 Change directory to run tests in. Default is 'st'.
41
42 =item I<--immediate>
43
44 Show errors as soon as they happen rather than at the end of the test run.
45                 
46 =item I<--target samba4|samba3|win>
47
48 Specify test target against which to run. Default is 'samba4'.
49
50 =item I<--quick>
51
52 Run only a limited number of tests. Intended to run in about 30 seconds on 
53 moderately recent systems.
54                 
55 =item I<--socket-wrapper>
56
57 Use socket wrapper library for communication with server. Only works 
58 when the server is running locally.
59
60 Will prevent TCP and UDP ports being opened on the local host but 
61 (transparently) redirects these calls to use unix domain sockets.
62
63 =item I<--expected-failures>
64
65 Specify a file containing a list of tests that are expected to fail. Failures for 
66 these tests will be counted as successes, successes will be counted as failures.
67
68 The format for the file is, one entry per line:
69
70 TESTSUITE-NAME/TEST-NAME
71
72 =item I<--skip>
73
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
76
77 =item I<--one>
78
79 Abort as soon as one test fails.
80
81 =back
82
83 =head1 ENVIRONMENT
84
85 =over 4
86
87 =item I<SMBD_VALGRIND>
88
89 =item I<TORTURE_MAXTIME>
90
91 =item I<VALGRIND>
92
93 =item I<TLS_ENABLED>
94
95 =item I<srcdir>
96
97 =back
98
99 =head1 LICENSE
100
101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
102
103 =head1 AUTHOR
104
105 Jelmer Vernooij
106
107 =cut
108
109 use strict;
110
111 use FindBin qw($RealBin $Script);
112 use File::Spec;
113 use Getopt::Long;
114 use POSIX;
115 use Cwd qw(abs_path);
116 use lib "$RealBin";
117 use env::Samba3;
118 use env::Samba4;
119 use env::Windows;
120 use SocketWrapper;
121
122 my $opt_help = 0;
123 my $opt_target = "samba4";
124 my $opt_quick = 0;
125 my $opt_socket_wrapper = 0;
126 my $opt_socket_wrapper_pcap = undef;
127 my $opt_socket_wrapper_keep_pcap = undef;
128 my $opt_one = 0;
129 my $opt_immediate = 0;
130 my $opt_expected_failures = undef;
131 my $opt_skip = undef;
132 my $opt_verbose = 0;
133 my $opt_testenv = 0;
134 my $ldap = undef;
135 my $opt_analyse_cmd = undef;
136 my $opt_resetup_env = undef;
137 my $opt_bindir = undef;
138 my $opt_no_lazy_setup = undef;
139 my $opt_format = "plain";
140
141 my $srcdir = ".";
142 my $builddir = ".";
143 my $prefix = "./st";
144
145 my @expected_failures = ();
146 my @skips = ();
147
148 my $statistics = {
149         START_TIME => time(),
150
151         SUITES_FAIL => 0,
152         SUITES_OK => 0,
153         SUITES_SKIPPED => 0,
154
155         TESTS_UNEXPECTED_OK => 0,
156         TESTS_EXPECTED_OK => 0,
157         TESTS_UNEXPECTED_FAIL => 0,
158         TESTS_EXPECTED_FAIL => 0,
159         TESTS_ERROR => 0
160 };
161
162 sub expecting_failure($)
163 {
164         my $fullname = shift;
165
166         foreach (@expected_failures) {
167                 return 1 if ($fullname =~ /$_/);
168         }
169
170         return 0;
171 }
172
173 sub skip($)
174 {
175         my $fullname = shift;
176
177         foreach (@skips) {
178                 return 1 if ($fullname =~ /$_/);
179         }
180
181         return 0;
182 }
183
184 sub getlog_env($);
185
186 sub setup_pcap($)
187 {
188         my ($state) = @_;
189
190         return unless ($opt_socket_wrapper_pcap);
191         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
192
193         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
194         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
195
196         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
197
198         SocketWrapper::setup_pcap($state->{PCAP_FILE});
199 }
200
201 sub cleanup_pcap($$$)
202 {
203         my ($state, $expected_ret, $ret) = @_;
204
205         return unless ($opt_socket_wrapper_pcap);
206         return if ($opt_socket_wrapper_keep_pcap);
207         return unless ($expected_ret == $ret);
208         return unless defined($state->{PCAP_FILE});
209
210         unlink($state->{PCAP_FILE});
211         $state->{PCAP_FILE} = undef;
212 }
213
214 sub parse_subunit_results($$$$)
215 {
216         my ($msg_ops, $msg_state, $statistics, $fh) = @_;
217         my $expected_ret = 1;
218         my $open_tests = {};
219
220         while(<$fh>) {
221                 $msg_ops->output_msg($msg_state, $_);
222                 if (/^test: (.+)\n/) {
223                         $open_tests->{$1} = 1;
224                         $msg_ops->start_test($msg_state, $1);
225                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
226                         my $result = $1;
227                         if ($1 eq "success") {
228                                 delete $open_tests->{$2};
229                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
230                                         $statistics->{TESTS_UNEXPECTED_OK}++;
231                                         $msg_ops->end_test($msg_state, $2, $1, 1);
232                                 } else {
233                                         $statistics->{TESTS_EXPECTED_OK}++;
234                                         $msg_ops->end_test($msg_state, $2, $1, 0);
235                                 }
236                         } elsif ($1 eq "failure") {
237                                 delete $open_tests->{$2};
238                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
239                                         $statistics->{TESTS_EXPECTED_FAIL}++;
240                                         $msg_ops->end_test($msg_state, $2, $1, 0);
241                                         $expected_ret = 0;
242                                 } else {
243                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
244                                         $msg_ops->end_test($msg_state, $2, $1, 1);
245                                 }
246                         } elsif ($1 eq "skip") {
247                                 delete $open_tests->{$2};
248                                 $msg_ops->end_test($msg_state, $2, $1, 0);
249                         } elsif ($1 eq "error") {
250                                 $statistics->{TESTS_ERROR}++;
251                                 delete $open_tests->{$2};
252                                 $msg_ops->end_test($msg_state, $2, $1, 1);
253                         }
254                 }
255         }
256
257         foreach (keys %$open_tests) {
258                 $msg_ops->end_test($msg_state, $_, "error", 1);
259                 $msg_ops->output_msg($msg_state, "$_ was started but never finished!");
260                 $statistics->{TESTS_ERROR}++;
261         }
262
263         return $expected_ret;
264 }
265
266 sub run_test($$$$$$)
267 {
268         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
269         my $msg_state = {
270                 ENVNAME => $envname,
271                 NAME    => $name,
272                 CMD     => $cmd,
273                 INDEX   => $i,
274                 TOTAL   => $totalsuites,
275                 START_TIME      => time()
276         };
277
278         setup_pcap($msg_state);
279
280         open(RESULT, "$cmd 2>&1|");
281         $msg_ops->start_testsuite($msg_state);
282
283         my $expected_ret = parse_subunit_results(
284                 $msg_ops, $msg_state, $statistics, *RESULT);
285
286         my $ret = close(RESULT);
287
288         cleanup_pcap($msg_state, $expected_ret, $ret);
289
290         $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
291                                                         getlog_env($msg_state->{ENVNAME}));
292
293         if (not $opt_socket_wrapper_keep_pcap and 
294                 defined($msg_state->{PCAP_FILE})) {
295                 $msg_ops->output_msg($msg_state, 
296                         "PCAP FILE: $msg_state->{PCAP_FILE}\n");
297         }
298
299         if ($ret != $expected_ret) {
300                 $statistics->{SUITES_FAIL}++;
301                 exit(1) if ($opt_one);
302         } else {
303                 $statistics->{SUITES_OK}++;
304         }
305
306         return ($ret == $expected_ret);
307 }
308
309 sub ShowHelp()
310 {
311         print "Samba test runner
312 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
313
314 Usage: $Script [OPTIONS] PREFIX
315
316 Generic options:
317  --help                     this help page
318  --target=samba4|samba3|win Samba version to target
319
320 Paths:
321  --prefix=DIR               prefix to run tests in [st]
322  --srcdir=DIR               source directory [.]
323  --builddir=DIR             output directory [.]
324
325 Target Specific:
326  --socket-wrapper-pcap=DIR      save traffic to pcap directories
327  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that 
328                             failed
329  --socket-wrapper           enable socket wrapper
330  --expected-failures=FILE   specify list of tests that is guaranteed to fail
331
332 Samba4 Specific:
333  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
334
335 Samba3 Specific:
336  --bindir=PATH              path to binaries
337
338 Behaviour:
339  --quick                    run quick overall test
340  --one                      abort when the first test fails
341  --immediate                print test output for failed tests during run
342  --verbose                  be verbose
343  --analyse-cmd CMD          command to run after each test
344 ";
345         exit(0);
346 }
347
348 my $result = GetOptions (
349                 'help|h|?' => \$opt_help,
350                 'target=s' => \$opt_target,
351                 'prefix=s' => \$prefix,
352                 'socket-wrapper' => \$opt_socket_wrapper,
353                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
354                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
355                 'quick' => \$opt_quick,
356                 'one' => \$opt_one,
357                 'immediate' => \$opt_immediate,
358                 'expected-failures=s' => \$opt_expected_failures,
359                 'skip=s' => \$opt_skip,
360                 'srcdir=s' => \$srcdir,
361                 'builddir=s' => \$builddir,
362                 'verbose' => \$opt_verbose,
363                 'testenv' => \$opt_testenv,
364                 'ldap:s' => \$ldap,
365                 'analyse-cmd=s' => \$opt_analyse_cmd,
366                 'no-lazy-setup' => \$opt_no_lazy_setup,
367                 'resetup-environment' => \$opt_resetup_env,
368                 'bindir:s' => \$opt_bindir,
369                 'format=s' => \$opt_format,
370             );
371
372 exit(1) if (not $result);
373
374 ShowHelp() if ($opt_help);
375
376 my $tests = shift;
377
378 # quick hack to disable rpc validation when using valgrind - its way too slow
379 unless (defined($ENV{VALGRIND})) {
380         $ENV{VALIDATE} = "validate";
381         $ENV{MALLOC_CHECK_} = 2;
382 }
383
384 my $old_pwd = "$RealBin/..";
385
386 # Backwards compatibility:
387 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
388         if (defined($ENV{FEDORA_DS_PREFIX})) {
389                 $ldap = "fedora-ds";
390         } else {
391                 $ldap = "openldap";
392         }
393 }
394
395 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
396 if ($ldap) {
397         # LDAP is slow
398         $torture_maxtime *= 2;
399 }
400
401 $prefix =~ s+//+/+;
402 $prefix =~ s+/./+/+;
403 $prefix =~ s+/$++;
404
405 die("using an empty prefix isn't allowed") unless $prefix ne "";
406
407 #Ensure we have the test prefix around
408 mkdir($prefix, 0777) unless -d $prefix;
409
410 my $prefix_abs = abs_path($prefix);
411 my $srcdir_abs = abs_path($srcdir);
412
413 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
414 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
415
416 $ENV{PREFIX} = $prefix;
417 $ENV{PREFIX_ABS} = $prefix_abs;
418 $ENV{SRCDIR} = $srcdir;
419 $ENV{SRCDIR_ABS} = $srcdir_abs;
420
421 my $tls_enabled = not $opt_quick;
422 if (defined($ENV{RUN_FROM_BUILD_FARM}) and 
423         ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
424         $opt_format = "buildfarm";
425 }
426
427 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
428 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
429 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
430 if (defined($ENV{LD_LIBRARY_PATH})) {
431         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
432 } else {
433         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
434 }
435 if (defined($ENV{PKG_CONFIG_PATH})) {
436         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
437 } else { 
438         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
439 }
440 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
441
442
443 if ($opt_socket_wrapper_pcap) {
444         # Socket wrapper pcap implies socket wrapper
445         $opt_socket_wrapper = 1;
446 }
447
448 my $socket_wrapper_dir;
449 if ($opt_socket_wrapper) {
450         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
451         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
452 } else {
453         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
454 }
455
456 my $target;
457
458 if ($opt_target eq "samba4") {
459         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
460 } elsif ($opt_target eq "samba3") {
461         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
462                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
463         }
464
465         $target = new Samba3($opt_bindir);
466 } elsif ($opt_target eq "win") {
467         die("Windows tests will not run with socket wrapper enabled.") 
468                 if ($opt_socket_wrapper);
469         $target = new Windows();
470 }
471
472 if (defined($opt_expected_failures)) {
473         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
474         while (<KNOWN>) { 
475                 chomp; 
476                 s/([ \t]+)\#(.*)$//;
477                 push (@expected_failures, $_); }
478         close(KNOWN);
479 }
480
481 if (defined($opt_skip)) {
482         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
483         while (<SKIP>) { 
484                 chomp; 
485                 s/([ \t]+)\#(.*)$//;
486                 push (@skips, $_); }
487         close(SKIP);
488 }
489
490 my $interfaces = join(',', ("127.0.0.6/8", 
491                             "127.0.0.7/8",
492                             "127.0.0.8/8",
493                             "127.0.0.9/8",
494                             "127.0.0.10/8",
495                             "127.0.0.11/8"));
496
497 my $conffile = "$prefix_abs/client/client.conf";
498
499 sub write_clientconf($$)
500 {
501         my ($conffile, $vars) = @_;
502
503         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
504         
505         if ( -d "$prefix/client/private" ) {
506                 unlink <$prefix/client/private/*>;
507         } else {
508                 mkdir("$prefix/client/private", 0777);
509         }
510
511         open(CF, ">$conffile");
512         print CF "[global]\n";
513         if (defined($ENV{VALGRIND})) {
514                 print CF "\ticonv:native = true\n";
515         } else {
516                 print CF "\ticonv:native = false\n";
517         }
518         print CF "\tnetbios name = client\n";
519         if (defined($vars->{DOMAIN})) {
520                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
521         }
522         if (defined($vars->{REALM})) {
523                 print CF "\trealm = $vars->{REALM}\n";
524         }
525         if (defined($vars->{NCALRPCDIR})) {
526                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
527         }
528         if (defined($vars->{PIDDIR})) {
529                 print CF "\tpid directory = $vars->{PIDDIR}\n";
530         }
531         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
532                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
533         }
534         print CF "
535         private dir = $prefix_abs/client/private
536         js include = $srcdir_abs/scripting/libjs
537         name resolve order = bcast
538         interfaces = $interfaces
539         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
540         max xmit = 32K
541         notify:inotify = false
542         ldb:nosync = true
543         system:anonymous = true
544         torture:basedir = $prefix_abs/client
545 #We don't want to pass our self-tests if the PAC code is wrong
546         gensec:require_pac = true
547 ";
548         close(CF);
549 }
550
551
552 my @torture_options = ();
553 push (@torture_options, "--configfile=$conffile");
554 # ensure any one smbtorture call doesn't run too long
555 push (@torture_options, "--maximum-runtime=$torture_maxtime");
556 push (@torture_options, "--target=$opt_target");
557 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
558 push (@torture_options, "--format=subunit");
559 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
560
561 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
562 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
563
564 my @todo = ();
565
566 my $testsdir = "$srcdir/selftest";
567 $ENV{SMB_CONF_PATH} = "$conffile";
568 $ENV{CONFIGURATION} = "--configfile=$conffile";
569
570 my %required_envs = ();
571
572 if ($opt_quick) {
573         open(IN, "$testsdir/tests_quick.sh|");
574 } else {
575         open(IN, "$testsdir/tests_all.sh|");
576 }
577 while (<IN>) {
578         if ($_ eq "-- TEST --\n") {
579                 my $name = <IN>;
580                 $name =~ s/\n//g;
581                 my $env = <IN>;
582                 $env =~ s/\n//g;
583                 my $cmdline = <IN>;
584                 $cmdline =~ s/\n//g;
585                 if (not defined($tests) or $name =~ /$tests/) {
586                         $required_envs{$env} = 1;
587                         push (@todo, [$name, $env, $cmdline]);
588                 }
589         } else {
590                 print;
591         }
592 }
593 close(IN) or die("Error creating recipe");
594
595 my $suitestotal = $#todo + 1;
596 my $i = 0;
597 $| = 1;
598
599 my %running_envs = ();
600
601 my @exported_envvars = (
602         # domain stuff
603         "DOMAIN",
604         "REALM",
605
606         # domain controller stuff
607         "DC_SERVER",
608         "DC_SERVER_IP",
609         "DC_NETBIOSNAME",
610         "DC_NETBIOSALIAS",
611
612         # server stuff
613         "SERVER",
614         "SERVER_IP",
615         "NETBIOSNAME",
616         "NETBIOSALIAS",
617
618         # user stuff
619         "USERNAME",
620         "PASSWORD",
621         "DC_USERNAME",
622         "DC_PASSWORD",
623
624         # misc stuff
625         "KRB5_CONFIG",
626         "WINBINDD_SOCKET_DIR"
627 );
628
629 sub setup_env($)
630 {
631         my ($envname) = @_;
632
633         my $testenv_vars;
634         if ($envname eq "none") {
635                 $testenv_vars = {};
636         } elsif (defined($running_envs{$envname})) {
637                 $testenv_vars = $running_envs{$envname};
638                 if (not $target->check_env($testenv_vars)) {
639                         $testenv_vars = undef;
640                 }
641         } else {
642                 $testenv_vars = $target->setup_env($envname, $prefix);
643         }
644
645         return undef unless defined($testenv_vars);
646
647         SocketWrapper::set_default_iface(6);
648         write_clientconf($conffile, $testenv_vars);
649
650         foreach (@exported_envvars) {
651                 if (defined($testenv_vars->{$_})) {
652                         $ENV{$_} = $testenv_vars->{$_};
653                 } else {
654                         delete $ENV{$_};
655                 }
656         }
657
658         $running_envs{$envname} = $testenv_vars;
659         return $testenv_vars;
660 }
661
662 sub exported_envvars_str($)
663 {
664         my ($testenv_vars) = @_;
665         my $out = "";
666
667         foreach (@exported_envvars) {
668                 next unless defined($testenv_vars->{$_});
669                 $out .= $_."=".$testenv_vars->{$_}."\n";
670         }
671
672         return $out;
673 }
674
675 sub getlog_env($)
676 {
677         my ($envname) = @_;
678         return "" if ($envname eq "none");
679         return $target->getlog_env($running_envs{$envname});
680 }
681
682 sub check_env($)
683 {
684         my ($envname) = @_;
685         return 1 if ($envname eq "none");
686         return $target->check_env($running_envs{$envname});
687 }
688
689 sub teardown_env($)
690 {
691         my ($envname) = @_;
692         return if ($envname eq "none");
693         $target->teardown_env($running_envs{$envname});
694         delete $running_envs{$envname};
695 }
696
697 my $msg_ops;
698 if ($opt_format eq "buildfarm") {
699         require output::buildfarm;
700         $msg_ops = new output::buildfarm();
701 } elsif ($opt_format eq "plain") {
702         require output::plain;
703         $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
704 } elsif ($opt_format eq "html") {
705         require output::html;
706         mkdir "test-results";
707         $msg_ops = new output::html("test-results", $statistics);
708 } else {
709         die("Invalid output format '$opt_format'");
710 }
711
712 if ($opt_no_lazy_setup) {
713         setup_env($_) foreach (keys %required_envs);
714 }
715
716 if ($opt_testenv) {
717         my $testenv_name = $ENV{SELFTEST_TESTENV};
718         $testenv_name = "dc" unless defined($testenv_name);
719
720         my $testenv_vars = setup_env($testenv_name);
721
722         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
723
724         my $envvarstr = exported_envvars_str($testenv_vars);
725
726         my $term = ($ENV{TERM} or "xterm");
727         system("$term -e 'echo -e \"
728 Welcome to the Samba4 Test environment '$testenv_name'
729
730 This matches the client environment used in make test
731 smbd is pid `cat \$PIDDIR/smbd.pid`
732
733 Some useful environment variables:
734 TORTURE_OPTIONS=\$TORTURE_OPTIONS
735 CONFIGURATION=\$CONFIGURATION
736
737 $envvarstr
738 \" && bash'");
739         teardown_env($testenv_name);
740 } else {
741         foreach (@todo) {
742                 $i++;
743                 my $cmd = $$_[2];
744                 $cmd =~ s/([\(\)])/\\$1/g;
745                 my $name = $$_[0];
746                 my $envname = $$_[1];
747                 
748                 if (skip($name)) {
749                         $msg_ops->skip_testsuite($name);
750                         $statistics->{SUITES_SKIPPED}++;
751                         next;
752                 }
753
754                 my $envvars = setup_env($envname);
755                 if (not defined($envvars)) {
756                         $statistics->{SUITES_FAIL}++;
757                         $statistics->{TESTS_ERROR}++;
758                         $msg_ops->missing_env($name, $envname);
759                         next;
760                 }
761
762                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
763
764                 if (defined($opt_analyse_cmd)) {
765                         system("$opt_analyse_cmd \"$name\"");
766                 }
767
768                 teardown_env($envname) if ($opt_resetup_env);
769         }
770 }
771
772 print "\n";
773
774 teardown_env($_) foreach (keys %running_envs);
775
776 $target->stop();
777
778 $statistics->{END_TIME} = time();
779 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
780 my $numfailed = $statistics->{SUITES_FAIL};
781 if ($numfailed == 0) {
782         my $ok = $statistics->{TESTS_EXPECTED_OK} + 
783                  $statistics->{TESTS_EXPECTED_FAIL};
784         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
785 } else {
786         $msg_ops->summary();
787 }
788 print "DURATION: $duration seconds\n";
789
790 my $failed = 0;
791
792 # if there were any valgrind failures, show them
793 foreach (<$prefix/valgrind.log*>) {
794         next unless (-s $_);
795         system("grep DWARF2.CFI.reader $_ > /dev/null");
796         if ($? >> 8 == 0) {
797             print "VALGRIND FAILURE\n";
798             $failed++;
799             system("cat $_");
800         }
801 }
802
803 if ($opt_format eq "buildfarm") {
804         print "TEST STATUS: $numfailed\n";
805 }
806
807 exit $numfailed;