r24671: More improvements to the HTML test suite results: hide
[kai/samba.git] / source4 / selftest / selftest.pl
1 #!/usr/bin/perl
2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
5
6 =pod
7
8 =head1 NAME
9
10 selftest - Samba test runner
11
12 =head1 SYNOPSIS
13
14 selftest --help
15
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
17
18 =head1 DESCRIPTION
19
20 A simple test runner. TESTS is a regular expression with tests to run.
21
22 =head1 OPTIONS
23
24 =over 4
25
26 =item I<--help>
27
28 Show list of available options.
29
30 =item I<--srcdir=DIR>
31
32 Source directory.
33
34 =item I<--builddir=DIR>
35
36 Build directory.
37
38 =item I<--prefix=DIR>
39
40 Change directory to run tests in. Default is 'st'.
41
42 =item I<--immediate>
43
44 Show errors as soon as they happen rather than at the end of the test run.
45                 
46 =item I<--target samba4|samba3|win>
47
48 Specify test target against which to run. Default is 'samba4'.
49
50 =item I<--quick>
51
52 Run only a limited number of tests. Intended to run in about 30 seconds on 
53 moderately recent systems.
54                 
55 =item I<--socket-wrapper>
56
57 Use socket wrapper library for communication with server. Only works 
58 when the server is running locally.
59
60 Will prevent TCP and UDP ports being opened on the local host but 
61 (transparently) redirects these calls to use unix domain sockets.
62
63 =item I<--expected-failures>
64
65 Specify a file containing a list of tests that are expected to fail. Failures for 
66 these tests will be counted as successes, successes will be counted as failures.
67
68 The format for the file is, one entry per line:
69
70 TESTSUITE-NAME/TEST-NAME
71
72 =item I<--skip>
73
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
76
77 =item I<--one>
78
79 Abort as soon as one test fails.
80
81 =back
82
83 =head1 ENVIRONMENT
84
85 =over 4
86
87 =item I<SMBD_VALGRIND>
88
89 =item I<TORTURE_MAXTIME>
90
91 =item I<VALGRIND>
92
93 =item I<TLS_ENABLED>
94
95 =item I<srcdir>
96
97 =back
98
99 =head1 LICENSE
100
101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
102
103 =head1 AUTHOR
104
105 Jelmer Vernooij
106
107 =cut
108
109 use strict;
110
111 use FindBin qw($RealBin $Script);
112 use File::Spec;
113 use Getopt::Long;
114 use POSIX;
115 use Cwd qw(abs_path);
116 use lib "$RealBin";
117 use env::Samba3;
118 use env::Samba4;
119 use env::Windows;
120 use SocketWrapper;
121
122 my $opt_help = 0;
123 my $opt_target = "samba4";
124 my $opt_quick = 0;
125 my $opt_socket_wrapper = 0;
126 my $opt_socket_wrapper_pcap = undef;
127 my $opt_socket_wrapper_keep_pcap = undef;
128 my $opt_one = 0;
129 my $opt_immediate = 0;
130 my $opt_expected_failures = undef;
131 my $opt_skip = undef;
132 my $opt_verbose = 0;
133 my $opt_testenv = 0;
134 my $ldap = undef;
135 my $opt_analyse_cmd = undef;
136 my $opt_resetup_env = undef;
137 my $opt_bindir = undef;
138 my $opt_no_lazy_setup = undef;
139 my $opt_format = "plain";
140
141 my $srcdir = ".";
142 my $builddir = ".";
143 my $prefix = "./st";
144
145 my @expected_failures = ();
146 my @skips = ();
147
148 my $statistics = {
149         START_TIME => time(),
150
151         SUITES_FAIL => 0,
152         SUITES_OK => 0,
153         SUITES_SKIPPED => 0,
154
155         TESTS_UNEXPECTED_OK => 0,
156         TESTS_EXPECTED_OK => 0,
157         TESTS_UNEXPECTED_FAIL => 0,
158         TESTS_EXPECTED_FAIL => 0,
159         TESTS_ERROR => 0,
160         TESTS_SKIP => 0,
161 };
162
163 sub expecting_failure($)
164 {
165         my $fullname = shift;
166
167         foreach (@expected_failures) {
168                 return 1 if ($fullname =~ /$_/);
169         }
170
171         return 0;
172 }
173
174 sub skip($)
175 {
176         my $fullname = shift;
177
178         foreach (@skips) {
179                 return 1 if ($fullname =~ /$_/);
180         }
181
182         return 0;
183 }
184
185 sub getlog_env($);
186
187 sub setup_pcap($)
188 {
189         my ($state) = @_;
190
191         return unless ($opt_socket_wrapper_pcap);
192         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
193
194         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
195         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
196
197         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
198
199         SocketWrapper::setup_pcap($state->{PCAP_FILE});
200 }
201
202 sub cleanup_pcap($$$)
203 {
204         my ($state, $expected_ret, $ret) = @_;
205
206         return unless ($opt_socket_wrapper_pcap);
207         return if ($opt_socket_wrapper_keep_pcap);
208         return unless ($expected_ret == $ret);
209         return unless defined($state->{PCAP_FILE});
210
211         unlink($state->{PCAP_FILE});
212         $state->{PCAP_FILE} = undef;
213 }
214
215 sub parse_subunit_results($$$$)
216 {
217         my ($msg_ops, $msg_state, $statistics, $fh) = @_;
218         my $expected_ret = 1;
219         my $open_tests = {};
220
221         while(<$fh>) {
222                 if (/^test: (.+)\n/) {
223                         $msg_ops->control_msg($msg_state, $_);
224                         $open_tests->{$1} = 1;
225                         $msg_ops->start_test($msg_state, $1);
226                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?([ \t]*)\n/) {
227                         $msg_ops->control_msg($msg_state, $_);
228                         my $reason = undef;
229                         if ($3) {
230                                 $reason = "";
231                                 # reason may be specified in next lines
232                                 while(<$fh>) {
233                                         $msg_ops->control_msg($msg_state, $_);
234                                         if ($_ eq "]\n") { last; } else { $reason .= $_; }
235                                 }
236                         }
237                         my $result = $1;
238                         if ($1 eq "success") {
239                                 delete $open_tests->{$2};
240                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
241                                         $statistics->{TESTS_UNEXPECTED_OK}++;
242                                         $msg_ops->end_test($msg_state, $2, $1, 1, $reason);
243                                 } else {
244                                         $statistics->{TESTS_EXPECTED_OK}++;
245                                         $msg_ops->end_test($msg_state, $2, $1, 0, $reason);
246                                 }
247                         } elsif ($1 eq "failure") {
248                                 delete $open_tests->{$2};
249                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
250                                         $statistics->{TESTS_EXPECTED_FAIL}++;
251                                         $msg_ops->end_test($msg_state, $2, $1, 0, $reason);
252                                         $expected_ret = 0;
253                                 } else {
254                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
255                                         $msg_ops->end_test($msg_state, $2, $1, 1, $reason);
256                                 }
257                         } elsif ($1 eq "skip") {
258                                 $statistics->{TESTS_SKIP}++;
259                                 delete $open_tests->{$2};
260                                 $msg_ops->end_test($msg_state, $2, $1, 0, $reason);
261                         } elsif ($1 eq "error") {
262                                 $statistics->{TESTS_ERROR}++;
263                                 delete $open_tests->{$2};
264                                 $msg_ops->end_test($msg_state, $2, $1, 1, $reason);
265                         }
266                 } else {
267                         $msg_ops->output_msg($msg_state, $_);
268                 }
269         }
270
271         foreach (keys %$open_tests) {
272                 $msg_ops->end_test($msg_state, $_, "error", 1, 
273                                                    "was started but never finished!");
274                 $statistics->{TESTS_ERROR}++;
275         }
276
277         return $expected_ret;
278 }
279
280 sub run_test($$$$$$)
281 {
282         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
283         my $msg_state = {
284                 ENVNAME => $envname,
285                 NAME    => $name,
286                 CMD     => $cmd,
287                 INDEX   => $i,
288                 TOTAL   => $totalsuites,
289                 START_TIME      => time()
290         };
291
292         setup_pcap($msg_state);
293
294         open(RESULT, "$cmd 2>&1|");
295         $msg_ops->start_testsuite($msg_state);
296
297         my $expected_ret = parse_subunit_results(
298                 $msg_ops, $msg_state, $statistics, *RESULT);
299
300         my $ret = close(RESULT);
301
302         cleanup_pcap($msg_state, $expected_ret, $ret);
303
304         $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
305                                                         getlog_env($msg_state->{ENVNAME}));
306
307         if (not $opt_socket_wrapper_keep_pcap and 
308                 defined($msg_state->{PCAP_FILE})) {
309                 $msg_ops->output_msg($msg_state, 
310                         "PCAP FILE: $msg_state->{PCAP_FILE}\n");
311         }
312
313         if ($ret != $expected_ret) {
314                 $statistics->{SUITES_FAIL}++;
315                 exit(1) if ($opt_one);
316         } else {
317                 $statistics->{SUITES_OK}++;
318         }
319
320         return ($ret == $expected_ret);
321 }
322
323 sub ShowHelp()
324 {
325         print "Samba test runner
326 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
327
328 Usage: $Script [OPTIONS] PREFIX
329
330 Generic options:
331  --help                     this help page
332  --target=samba4|samba3|win Samba version to target
333
334 Paths:
335  --prefix=DIR               prefix to run tests in [st]
336  --srcdir=DIR               source directory [.]
337  --builddir=DIR             output directory [.]
338
339 Target Specific:
340  --socket-wrapper-pcap=DIR      save traffic to pcap directories
341  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that 
342                             failed
343  --socket-wrapper           enable socket wrapper
344  --expected-failures=FILE   specify list of tests that is guaranteed to fail
345
346 Samba4 Specific:
347  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
348
349 Samba3 Specific:
350  --bindir=PATH              path to binaries
351
352 Behaviour:
353  --quick                    run quick overall test
354  --one                      abort when the first test fails
355  --immediate                print test output for failed tests during run
356  --verbose                  be verbose
357  --analyse-cmd CMD          command to run after each test
358 ";
359         exit(0);
360 }
361
362 my $result = GetOptions (
363                 'help|h|?' => \$opt_help,
364                 'target=s' => \$opt_target,
365                 'prefix=s' => \$prefix,
366                 'socket-wrapper' => \$opt_socket_wrapper,
367                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
368                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
369                 'quick' => \$opt_quick,
370                 'one' => \$opt_one,
371                 'immediate' => \$opt_immediate,
372                 'expected-failures=s' => \$opt_expected_failures,
373                 'skip=s' => \$opt_skip,
374                 'srcdir=s' => \$srcdir,
375                 'builddir=s' => \$builddir,
376                 'verbose' => \$opt_verbose,
377                 'testenv' => \$opt_testenv,
378                 'ldap:s' => \$ldap,
379                 'analyse-cmd=s' => \$opt_analyse_cmd,
380                 'no-lazy-setup' => \$opt_no_lazy_setup,
381                 'resetup-environment' => \$opt_resetup_env,
382                 'bindir:s' => \$opt_bindir,
383                 'format=s' => \$opt_format,
384             );
385
386 exit(1) if (not $result);
387
388 ShowHelp() if ($opt_help);
389
390 my $tests = shift;
391
392 # quick hack to disable rpc validation when using valgrind - its way too slow
393 unless (defined($ENV{VALGRIND})) {
394         $ENV{VALIDATE} = "validate";
395         $ENV{MALLOC_CHECK_} = 2;
396 }
397
398 my $old_pwd = "$RealBin/..";
399
400 # Backwards compatibility:
401 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
402         if (defined($ENV{FEDORA_DS_PREFIX})) {
403                 $ldap = "fedora-ds";
404         } else {
405                 $ldap = "openldap";
406         }
407 }
408
409 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
410 if ($ldap) {
411         # LDAP is slow
412         $torture_maxtime *= 2;
413 }
414
415 $prefix =~ s+//+/+;
416 $prefix =~ s+/./+/+;
417 $prefix =~ s+/$++;
418
419 die("using an empty prefix isn't allowed") unless $prefix ne "";
420
421 #Ensure we have the test prefix around
422 mkdir($prefix, 0777) unless -d $prefix;
423
424 my $prefix_abs = abs_path($prefix);
425 my $srcdir_abs = abs_path($srcdir);
426
427 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
428 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
429
430 $ENV{PREFIX} = $prefix;
431 $ENV{PREFIX_ABS} = $prefix_abs;
432 $ENV{SRCDIR} = $srcdir;
433 $ENV{SRCDIR_ABS} = $srcdir_abs;
434
435 my $tls_enabled = not $opt_quick;
436 if (defined($ENV{RUN_FROM_BUILD_FARM}) and 
437         ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
438         $opt_format = "buildfarm";
439 }
440
441 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
442 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
443 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
444 if (defined($ENV{LD_LIBRARY_PATH})) {
445         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
446 } else {
447         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
448 }
449 if (defined($ENV{PKG_CONFIG_PATH})) {
450         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
451 } else { 
452         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
453 }
454 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
455
456
457 if ($opt_socket_wrapper_pcap) {
458         # Socket wrapper pcap implies socket wrapper
459         $opt_socket_wrapper = 1;
460 }
461
462 my $socket_wrapper_dir;
463 if ($opt_socket_wrapper) {
464         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
465         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
466 } else {
467         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
468 }
469
470 my $target;
471
472 if ($opt_target eq "samba4") {
473         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
474 } elsif ($opt_target eq "samba3") {
475         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
476                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
477         }
478
479         $target = new Samba3($opt_bindir);
480 } elsif ($opt_target eq "win") {
481         die("Windows tests will not run with socket wrapper enabled.") 
482                 if ($opt_socket_wrapper);
483         $target = new Windows();
484 }
485
486 if (defined($opt_expected_failures)) {
487         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
488         while (<KNOWN>) { 
489                 chomp; 
490                 s/([ \t]+)\#(.*)$//;
491                 push (@expected_failures, $_); }
492         close(KNOWN);
493 }
494
495 if (defined($opt_skip)) {
496         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
497         while (<SKIP>) { 
498                 chomp; 
499                 s/([ \t]+)\#(.*)$//;
500                 push (@skips, $_); }
501         close(SKIP);
502 }
503
504 my $interfaces = join(',', ("127.0.0.6/8", 
505                             "127.0.0.7/8",
506                             "127.0.0.8/8",
507                             "127.0.0.9/8",
508                             "127.0.0.10/8",
509                             "127.0.0.11/8"));
510
511 my $conffile = "$prefix_abs/client/client.conf";
512
513 sub write_clientconf($$)
514 {
515         my ($conffile, $vars) = @_;
516
517         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
518         
519         if ( -d "$prefix/client/private" ) {
520                 unlink <$prefix/client/private/*>;
521         } else {
522                 mkdir("$prefix/client/private", 0777);
523         }
524
525         open(CF, ">$conffile");
526         print CF "[global]\n";
527         if (defined($ENV{VALGRIND})) {
528                 print CF "\ticonv:native = true\n";
529         } else {
530                 print CF "\ticonv:native = false\n";
531         }
532         print CF "\tnetbios name = client\n";
533         if (defined($vars->{DOMAIN})) {
534                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
535         }
536         if (defined($vars->{REALM})) {
537                 print CF "\trealm = $vars->{REALM}\n";
538         }
539         if (defined($vars->{NCALRPCDIR})) {
540                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
541         }
542         if (defined($vars->{PIDDIR})) {
543                 print CF "\tpid directory = $vars->{PIDDIR}\n";
544         }
545         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
546                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
547         }
548         print CF "
549         private dir = $prefix_abs/client/private
550         js include = $srcdir_abs/scripting/libjs
551         name resolve order = bcast
552         interfaces = $interfaces
553         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
554         max xmit = 32K
555         notify:inotify = false
556         ldb:nosync = true
557         system:anonymous = true
558         torture:basedir = $prefix_abs/client
559 #We don't want to pass our self-tests if the PAC code is wrong
560         gensec:require_pac = true
561 ";
562         close(CF);
563 }
564
565
566 my @torture_options = ();
567 push (@torture_options, "--configfile=$conffile");
568 # ensure any one smbtorture call doesn't run too long
569 push (@torture_options, "--maximum-runtime=$torture_maxtime");
570 push (@torture_options, "--target=$opt_target");
571 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
572 push (@torture_options, "--format=subunit");
573 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
574
575 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
576 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
577
578 my @todo = ();
579
580 my $testsdir = "$srcdir/selftest";
581 $ENV{SMB_CONF_PATH} = "$conffile";
582 $ENV{CONFIGURATION} = "--configfile=$conffile";
583
584 my %required_envs = ();
585
586 if ($opt_quick) {
587         open(IN, "$testsdir/tests_quick.sh|");
588 } else {
589         open(IN, "$testsdir/tests_all.sh|");
590 }
591 while (<IN>) {
592         if ($_ eq "-- TEST --\n") {
593                 my $name = <IN>;
594                 $name =~ s/\n//g;
595                 my $env = <IN>;
596                 $env =~ s/\n//g;
597                 my $cmdline = <IN>;
598                 $cmdline =~ s/\n//g;
599                 if (not defined($tests) or $name =~ /$tests/) {
600                         $required_envs{$env} = 1;
601                         push (@todo, [$name, $env, $cmdline]);
602                 }
603         } else {
604                 print;
605         }
606 }
607 close(IN) or die("Error creating recipe");
608
609 my $suitestotal = $#todo + 1;
610 my $i = 0;
611 $| = 1;
612
613 my %running_envs = ();
614
615 my @exported_envvars = (
616         # domain stuff
617         "DOMAIN",
618         "REALM",
619
620         # domain controller stuff
621         "DC_SERVER",
622         "DC_SERVER_IP",
623         "DC_NETBIOSNAME",
624         "DC_NETBIOSALIAS",
625
626         # server stuff
627         "SERVER",
628         "SERVER_IP",
629         "NETBIOSNAME",
630         "NETBIOSALIAS",
631
632         # user stuff
633         "USERNAME",
634         "PASSWORD",
635         "DC_USERNAME",
636         "DC_PASSWORD",
637
638         # misc stuff
639         "KRB5_CONFIG",
640         "WINBINDD_SOCKET_DIR"
641 );
642
643 sub setup_env($)
644 {
645         my ($envname) = @_;
646
647         my $testenv_vars;
648         if ($envname eq "none") {
649                 $testenv_vars = {};
650         } elsif (defined($running_envs{$envname})) {
651                 $testenv_vars = $running_envs{$envname};
652                 if (not $target->check_env($testenv_vars)) {
653                         $testenv_vars = undef;
654                 }
655         } else {
656                 $testenv_vars = $target->setup_env($envname, $prefix);
657         }
658
659         return undef unless defined($testenv_vars);
660
661         SocketWrapper::set_default_iface(6);
662         write_clientconf($conffile, $testenv_vars);
663
664         foreach (@exported_envvars) {
665                 if (defined($testenv_vars->{$_})) {
666                         $ENV{$_} = $testenv_vars->{$_};
667                 } else {
668                         delete $ENV{$_};
669                 }
670         }
671
672         $running_envs{$envname} = $testenv_vars;
673         return $testenv_vars;
674 }
675
676 sub exported_envvars_str($)
677 {
678         my ($testenv_vars) = @_;
679         my $out = "";
680
681         foreach (@exported_envvars) {
682                 next unless defined($testenv_vars->{$_});
683                 $out .= $_."=".$testenv_vars->{$_}."\n";
684         }
685
686         return $out;
687 }
688
689 sub getlog_env($)
690 {
691         my ($envname) = @_;
692         return "" if ($envname eq "none");
693         return $target->getlog_env($running_envs{$envname});
694 }
695
696 sub check_env($)
697 {
698         my ($envname) = @_;
699         return 1 if ($envname eq "none");
700         return $target->check_env($running_envs{$envname});
701 }
702
703 sub teardown_env($)
704 {
705         my ($envname) = @_;
706         return if ($envname eq "none");
707         $target->teardown_env($running_envs{$envname});
708         delete $running_envs{$envname};
709 }
710
711 my $msg_ops;
712 if ($opt_format eq "buildfarm") {
713         require output::buildfarm;
714         $msg_ops = new output::buildfarm();
715 } elsif ($opt_format eq "plain") {
716         require output::plain;
717         $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
718 } elsif ($opt_format eq "html") {
719         require output::html;
720         mkdir "test-results";
721         $msg_ops = new output::html("test-results", $statistics);
722 } else {
723         die("Invalid output format '$opt_format'");
724 }
725
726 if ($opt_no_lazy_setup) {
727         setup_env($_) foreach (keys %required_envs);
728 }
729
730 if ($opt_testenv) {
731         my $testenv_name = $ENV{SELFTEST_TESTENV};
732         $testenv_name = "dc" unless defined($testenv_name);
733
734         my $testenv_vars = setup_env($testenv_name);
735
736         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
737
738         my $envvarstr = exported_envvars_str($testenv_vars);
739
740         my $term = ($ENV{TERM} or "xterm");
741         system("$term -e 'echo -e \"
742 Welcome to the Samba4 Test environment '$testenv_name'
743
744 This matches the client environment used in make test
745 smbd is pid `cat \$PIDDIR/smbd.pid`
746
747 Some useful environment variables:
748 TORTURE_OPTIONS=\$TORTURE_OPTIONS
749 CONFIGURATION=\$CONFIGURATION
750
751 $envvarstr
752 \" && bash'");
753         teardown_env($testenv_name);
754 } else {
755         foreach (@todo) {
756                 $i++;
757                 my $cmd = $$_[2];
758                 $cmd =~ s/([\(\)])/\\$1/g;
759                 my $name = $$_[0];
760                 my $envname = $$_[1];
761                 
762                 if (skip($name)) {
763                         $msg_ops->skip_testsuite($name);
764                         $statistics->{SUITES_SKIPPED}++;
765                         next;
766                 }
767
768                 my $envvars = setup_env($envname);
769                 if (not defined($envvars)) {
770                         $statistics->{SUITES_FAIL}++;
771                         $statistics->{TESTS_ERROR}++;
772                         $msg_ops->missing_env($name, $envname);
773                         next;
774                 }
775
776                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
777
778                 if (defined($opt_analyse_cmd)) {
779                         system("$opt_analyse_cmd \"$name\"");
780                 }
781
782                 teardown_env($envname) if ($opt_resetup_env);
783         }
784 }
785
786 print "\n";
787
788 teardown_env($_) foreach (keys %running_envs);
789
790 $target->stop();
791
792 $statistics->{END_TIME} = time();
793 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
794 $msg_ops->summary();
795 print "DURATION: $duration seconds\n";
796
797 my $failed = 0;
798
799 # if there were any valgrind failures, show them
800 foreach (<$prefix/valgrind.log*>) {
801         next unless (-s $_);
802         system("grep DWARF2.CFI.reader $_ > /dev/null");
803         if ($? >> 8 == 0) {
804             print "VALGRIND FAILURE\n";
805             $failed++;
806             system("cat $_");
807         }
808 }
809
810 if ($opt_format eq "buildfarm") {
811         print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
812 }
813
814 exit $statistics->{SUITES_FAIL};