r24338: Add callbacks for starting and finishing tests.
[garming/samba-autobuild/.git] / source4 / selftest / selftest.pl
1 #!/usr/bin/perl
2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
5
6 =pod
7
8 =head1 NAME
9
10 selftest - Samba test runner
11
12 =head1 SYNOPSIS
13
14 selftest --help
15
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
17
18 =head1 DESCRIPTION
19
20 A simple test runner. TESTS is a regular expression with tests to run.
21
22 =head1 OPTIONS
23
24 =over 4
25
26 =item I<--help>
27
28 Show list of available options.
29
30 =item I<--srcdir=DIR>
31
32 Source directory.
33
34 =item I<--builddir=DIR>
35
36 Build directory.
37
38 =item I<--prefix=DIR>
39
40 Change directory to run tests in. Default is 'st'.
41
42 =item I<--immediate>
43
44 Show errors as soon as they happen rather than at the end of the test run.
45                 
46 =item I<--target samba4|samba3|win>
47
48 Specify test target against which to run. Default is 'samba4'.
49
50 =item I<--quick>
51
52 Run only a limited number of tests. Intended to run in about 30 seconds on 
53 moderately recent systems.
54                 
55 =item I<--socket-wrapper>
56
57 Use socket wrapper library for communication with server. Only works 
58 when the server is running locally.
59
60 Will prevent TCP and UDP ports being opened on the local host but 
61 (transparently) redirects these calls to use unix domain sockets.
62
63 =item I<--expected-failures>
64
65 Specify a file containing a list of tests that are expected to fail. Failures for 
66 these tests will be counted as successes, successes will be counted as failures.
67
68 The format for the file is, one entry per line:
69
70 TESTSUITE-NAME/TEST-NAME
71
72 =item I<--skip>
73
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
76
77 =item I<--one>
78
79 Abort as soon as one test fails.
80
81 =back
82
83 =head1 ENVIRONMENT
84
85 =over 4
86
87 =item I<SMBD_VALGRIND>
88
89 =item I<TORTURE_MAXTIME>
90
91 =item I<VALGRIND>
92
93 =item I<TLS_ENABLED>
94
95 =item I<srcdir>
96
97 =back
98
99 =head1 LICENSE
100
101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
102
103 =head1 AUTHOR
104
105 Jelmer Vernooij
106
107 =cut
108
109 use strict;
110
111 use FindBin qw($RealBin $Script);
112 use File::Spec;
113 use Getopt::Long;
114 use POSIX;
115 use Cwd qw(abs_path);
116 use lib "$RealBin";
117 use Samba3;
118 use Samba4;
119 use SocketWrapper;
120
121 my $opt_help = 0;
122 my $opt_target = "samba4";
123 my $opt_quick = 0;
124 my $opt_socket_wrapper = 0;
125 my $opt_socket_wrapper_pcap = undef;
126 my $opt_socket_wrapper_keep_pcap = undef;
127 my $opt_one = 0;
128 my $opt_immediate = 0;
129 my $opt_expected_failures = undef;
130 my $opt_skip = undef;
131 my $opt_verbose = 0;
132 my $opt_testenv = 0;
133 my $ldap = undef;
134 my $opt_analyse_cmd = undef;
135 my $opt_resetup_env = undef;
136 my $opt_bindir = undef;
137 my $opt_no_lazy_setup = undef;
138
139 my $srcdir = ".";
140 my $builddir = ".";
141 my $prefix = "./st";
142
143 my $suitesfailed = [];
144 my $start = time();
145 my @expected_failures = ();
146 my @skips = ();
147
148 my $statistics = {
149         SUITES_FAIL => 0,
150         SUITES_OK => 0,
151         SUITES_SKIPPED => 0,
152
153         TESTS_UNEXPECTED_OK => 0,
154         TESTS_EXPECTED_OK => 0,
155         TESTS_UNEXPECTED_FAIL => 0,
156         TESTS_EXPECTED_FAIL => 0,
157         TESTS_ERROR => 0
158 };
159
160 sub expecting_failure($)
161 {
162         my $fullname = shift;
163
164         foreach (@expected_failures) {
165                 return 1 if ($fullname =~ /$_/);
166         }
167
168         return 0;
169 }
170
171 sub skip($)
172 {
173         my $fullname = shift;
174
175         foreach (@skips) {
176                 return 1 if ($fullname =~ /$_/);
177         }
178
179         return 0;
180 }
181
182 sub getlog_env($);
183
184 my $test_output = {};
185
186 sub buildfarm_start_testsuite($)
187 {
188         my ($state) = @_;
189         my $out = "";
190
191         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
192         $out .= "Running test $state->{NAME} (level 0 stdout)\n";
193         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
194         $out .= scalar(localtime())."\n";
195         $out .= "SELFTEST RUNTIME: " . ($state->{START} - $start) . "s\n";
196         $out .= "NAME: $state->{NAME}\n";
197         $out .= "CMD: $state->{CMD}\n";
198
199         $test_output->{$state->{NAME}} = "";
200
201         print $out;
202 }
203
204 sub buildfarm_output_msg($$)
205 {
206         my ($state, $output) = @_;
207
208         $test_output->{$state->{NAME}} .= $output;
209 }
210
211 sub buildfarm_end_testsuite($$$)
212 {
213         my ($state, $expected_ret, $ret) = @_;
214         my $out = "";
215
216         $out .= "TEST RUNTIME: " . (time() - $state->{START}) . "s\n";
217
218         if ($ret == $expected_ret) {
219                 $out .= "ALL OK\n";
220         } else {
221                 $out .= "ERROR: $ret";
222                 $out .= $test_output->{$state->{NAME}};
223         }
224
225         $out .= "PCAP FILE: $state->{PCAP_FILE}\n" if defined($state->{PCAP_FILE});
226
227         $out .= getlog_env($state->{ENVNAME});
228
229         $out .= "==========================================\n";
230         if ($ret == $expected_ret) {
231                 $out .= "TEST PASSED: $state->{NAME}\n";
232         } else {
233                 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
234         }
235         $out .= "==========================================\n";
236
237         print $out;
238 }
239
240 sub buildfarm_start_test($$)
241 {
242         my ($state, $testname) = @_;
243 }
244
245 sub buildfarm_end_test($$$$)
246 {
247         my ($state, $testname, $result, $expected) = @_;
248 }
249
250 my $buildfarm_msg_ops = {
251         start_testsuite => \&buildfarm_start_testsuite,
252         output_msg      => \&buildfarm_output_msg,
253         end_testsuite           => \&buildfarm_end_testsuite,
254         start_test => \&buildfarm_start_test,
255         end_test => \&buildfarm_end_test,
256 };
257
258 sub plain_output_msg($$);
259
260 sub plain_start_testsuite($)
261 {
262         my ($state) = @_;
263         my $out = "";
264
265         $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
266         $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
267         $out .= "] $state->{NAME}\n";
268
269         $test_output->{$state->{NAME}} = "" unless $opt_verbose;
270
271         plain_output_msg($state, "CMD: $state->{CMD}\n");
272
273         print $out;
274 }
275
276 sub plain_output_msg($$)
277 {
278         my ($state, $output) = @_;
279
280         if ($opt_verbose) {
281                 print $output;
282         } else {
283                 $test_output->{$state->{NAME}} .= $output;
284         }
285 }
286
287 sub plain_end_testsuite($$$)
288 {
289         my ($state, $expected_ret, $ret) = @_;
290         my $out = "";
291
292         if ($ret != $expected_ret) {
293                 plain_output_msg($state, "ERROR: $ret\n");
294         }
295
296         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
297                 $out .= $test_output->{$state->{NAME}};
298         }
299
300         if (not $opt_socket_wrapper_keep_pcap and defined($state->{PCAP_FILE})) {
301                 $out .= "PCAP FILE: $state->{PCAP_FILE}\n";
302         }
303
304         $out .= getlog_env($state->{ENVNAME});
305
306         print $out;
307 }
308
309 sub plain_start_test($$)
310 {
311         my ($state, $testname) = @_;
312 }
313
314 sub plain_end_test($$$$)
315 {
316         my ($state, $testname, $result, $expected) = @_;
317 }
318
319 my $plain_msg_ops = {
320         start_testsuite => \&plain_start_testsuite,
321         output_msg      => \&plain_output_msg,
322         end_testsuite           => \&plain_end_testsuite,
323         start_test => \&plain_start_test,
324         end_test => \&plain_end_test,
325 };
326
327 sub setup_pcap($)
328 {
329         my ($state) = @_;
330
331         return unless ($opt_socket_wrapper_pcap);
332         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
333
334         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
335         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
336
337         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
338
339         SocketWrapper::setup_pcap($state->{PCAP_FILE});
340 }
341
342 sub cleanup_pcap($$$)
343 {
344         my ($state, $expected_ret, $ret) = @_;
345
346         return unless ($opt_socket_wrapper_pcap);
347         return if ($opt_socket_wrapper_keep_pcap);
348         return unless ($expected_ret == $ret);
349         return unless defined($state->{PCAP_FILE});
350
351         unlink($state->{PCAP_FILE});
352         $state->{PCAP_FILE} = undef;
353 }
354
355 sub run_test($$$$$$)
356 {
357         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
358         my $expected_ret = 1;
359         my $open_tests = {};
360         my $msg_state = {
361                 ENVNAME => $envname,
362                 NAME    => $name,
363                 CMD     => $cmd,
364                 INDEX   => $i,
365                 TOTAL   => $totalsuites,
366                 START   => time()
367         };
368
369         setup_pcap($msg_state);
370
371         $msg_ops->{start_testsuite}->($msg_state);
372
373         open(RESULT, "$cmd 2>&1|");
374         while (<RESULT>) {
375                 $msg_ops->{output_msg}->($msg_state, $_);
376                 if (/^test: (.+)\n/) {
377                         $open_tests->{$1} = 1;
378                         $msg_ops->{start_test}->($msg_state, $1);
379                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
380                         my $result = $1;
381                         if ($1 eq "success") {
382                                 delete $open_tests->{$2};
383                                 if (expecting_failure("$name/$2")) {
384                                         $statistics->{TESTS_UNEXPECTED_OK}++;
385                                         $msg_ops->{end_test}->($msg_state, $2, $1, 1);
386                                 } else {
387                                         $statistics->{TESTS_EXPECTED_OK}++;
388                                         $msg_ops->{end_test}->($msg_state, $2, $1, 0);
389                                 }
390                         } elsif ($1 eq "failure") {
391                                 delete $open_tests->{$2};
392                                 if (expecting_failure("$name/$2")) {
393                                         $statistics->{TESTS_EXPECTED_FAIL}++;
394                                         $msg_ops->{end_test}->($msg_state, $2, $1, 0);
395                                         $expected_ret = 0;
396                                 } else {
397                                         print "n:$name/$2\n";
398                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
399                                         $msg_ops->{end_test}->($msg_state, $2, $1, 1);
400                                 }
401                         } elsif ($1 eq "skip") {
402                                 delete $open_tests->{$2};
403                                 $msg_ops->{end_test}->($msg_state, $2, $1, 0);
404                         } elsif ($1 eq "error") {
405                                 $statistics->{TESTS_ERROR}++;
406                                 delete $open_tests->{$2};
407                                 $msg_ops->{end_test}->($msg_state, $2, $1, 1);
408                         }
409                 }
410         }
411
412         foreach (keys %$open_tests) {
413                 $msg_ops->{end_test}->($msg_state, $_, "error", 1);
414                 $msg_ops->{output_msg}->($msg_state, "$_ was started but never finished!");
415                 $statistics->{TESTS_ERROR}++;
416         }
417         my $ret = close(RESULT);
418
419         cleanup_pcap($msg_state,  $expected_ret, $ret);
420
421         $msg_ops->{end_testsuite}->($msg_state, $expected_ret, $ret);
422
423         if ($ret != $expected_ret) {
424                 push(@$suitesfailed, $name);
425                 $statistics->{SUITES_FAIL}++;
426                 exit(1) if ($opt_one);
427         } else {
428                 $statistics->{SUITES_OK}++;
429         }
430
431         return ($ret == $expected_ret);
432 }
433
434 sub ShowHelp()
435 {
436         print "Samba test runner
437 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
438
439 Usage: $Script [OPTIONS] PREFIX
440
441 Generic options:
442  --help                     this help page
443  --target=samba4|samba3|win Samba version to target
444
445 Paths:
446  --prefix=DIR               prefix to run tests in [st]
447  --srcdir=DIR               source directory [.]
448  --builddir=DIR             output directory [.]
449
450 Target Specific:
451  --socket-wrapper-pcap=DIR      save traffic to pcap directories
452  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that 
453                             failed
454  --socket-wrapper           enable socket wrapper
455  --expected-failures=FILE   specify list of tests that is guaranteed to fail
456
457 Samba4 Specific:
458  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
459
460 Samba3 Specific:
461  --bindir=PATH              path to binaries
462
463 Behaviour:
464  --quick                    run quick overall test
465  --one                      abort when the first test fails
466  --immediate                print test output for failed tests during run
467  --verbose                  be verbose
468  --analyse-cmd CMD          command to run after each test
469 ";
470         exit(0);
471 }
472
473 my $result = GetOptions (
474                 'help|h|?' => \$opt_help,
475                 'target=s' => \$opt_target,
476                 'prefix=s' => \$prefix,
477                 'socket-wrapper' => \$opt_socket_wrapper,
478                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
479                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
480                 'quick' => \$opt_quick,
481                 'one' => \$opt_one,
482                 'immediate' => \$opt_immediate,
483                 'expected-failures=s' => \$opt_expected_failures,
484                 'skip=s' => \$opt_skip,
485                 'srcdir=s' => \$srcdir,
486                 'builddir=s' => \$builddir,
487                 'verbose' => \$opt_verbose,
488                 'testenv' => \$opt_testenv,
489                 'ldap:s' => \$ldap,
490                 'analyse-cmd=s' => \$opt_analyse_cmd,
491                 'no-lazy-setup' => \$opt_no_lazy_setup,
492                 'resetup-environment' => \$opt_resetup_env,
493                 'bindir:s' => \$opt_bindir,
494             );
495
496 exit(1) if (not $result);
497
498 ShowHelp() if ($opt_help);
499
500 my $tests = shift;
501
502 # quick hack to disable rpc validation when using valgrind - its way too slow
503 unless (defined($ENV{VALGRIND})) {
504         $ENV{VALIDATE} = "validate";
505         $ENV{MALLOC_CHECK_} = 2;
506 }
507
508 my $old_pwd = "$RealBin/..";
509
510 # Backwards compatibility:
511 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
512         if (defined($ENV{FEDORA_DS_PREFIX})) {
513                 $ldap = "fedora-ds";
514         } else {
515                 $ldap = "openldap";
516         }
517 }
518
519 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
520 if ($ldap) {
521         # LDAP is slow
522         $torture_maxtime *= 2;
523 }
524
525 $prefix =~ s+//+/+;
526 $prefix =~ s+/./+/+;
527 $prefix =~ s+/$++;
528
529 die("using an empty prefix isn't allowed") unless $prefix ne "";
530
531 #Ensure we have the test prefix around
532 mkdir($prefix, 0777) unless -d $prefix;
533
534 my $prefix_abs = abs_path($prefix);
535 my $srcdir_abs = abs_path($srcdir);
536
537 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
538 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
539
540 $ENV{PREFIX} = $prefix;
541 $ENV{PREFIX_ABS} = $prefix_abs;
542 $ENV{SRCDIR} = $srcdir;
543 $ENV{SRCDIR_ABS} = $srcdir_abs;
544
545 my $tls_enabled = not $opt_quick;
546 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and 
547                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
548
549 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
550 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
551 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
552 if (defined($ENV{LD_LIBRARY_PATH})) {
553         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
554 } else {
555         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
556 }
557 if (defined($ENV{PKG_CONFIG_PATH})) {
558         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
559 } else { 
560         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
561 }
562 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
563
564
565 if ($opt_socket_wrapper_pcap) {
566         # Socket wrapper pcap implies socket wrapper
567         $opt_socket_wrapper = 1;
568 }
569
570 my $socket_wrapper_dir;
571 if ($opt_socket_wrapper) {
572         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
573         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
574 } else {
575         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
576 }
577
578 my $target;
579
580 if ($opt_target eq "samba4") {
581         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
582 } elsif ($opt_target eq "samba3") {
583         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
584                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
585         }
586
587         $target = new Samba3($opt_bindir);
588 } elsif ($opt_target eq "win") {
589         die("Windows tests will not run with socket wrapper enabled.") 
590                 if ($opt_socket_wrapper);
591         $target = new Windows();
592 }
593
594 if (defined($opt_expected_failures)) {
595         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
596         while (<KNOWN>) { 
597                 chomp; 
598                 s/([ \t]+)\#(.*)$//;
599                 push (@expected_failures, $_); }
600         close(KNOWN);
601 }
602
603 if (defined($opt_skip)) {
604         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
605         while (<SKIP>) { 
606                 chomp; 
607                 s/([ \t]+)\#(.*)$//;
608                 push (@skips, $_); }
609         close(SKIP);
610 }
611
612 my $interfaces = join(',', ("127.0.0.6/8", 
613                             "127.0.0.7/8",
614                             "127.0.0.8/8",
615                             "127.0.0.9/8",
616                             "127.0.0.10/8",
617                             "127.0.0.11/8"));
618
619 my $conffile = "$prefix_abs/client/client.conf";
620
621 sub write_clientconf($$)
622 {
623         my ($conffile, $vars) = @_;
624
625         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
626         
627         if ( -d "$prefix/client/private" ) {
628                 unlink <$prefix/client/private/*>;
629         } else {
630                 mkdir("$prefix/client/private", 0777);
631         }
632
633         open(CF, ">$conffile");
634         print CF "[global]\n";
635         if (defined($ENV{VALGRIND})) {
636                 print CF "\ticonv:native = true\n";
637         } else {
638                 print CF "\ticonv:native = false\n";
639         }
640         print CF "\tnetbios name = client\n";
641         if (defined($vars->{DOMAIN})) {
642                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
643         }
644         if (defined($vars->{REALM})) {
645                 print CF "\trealm = $vars->{REALM}\n";
646         }
647         if (defined($vars->{NCALRPCDIR})) {
648                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
649         }
650         if (defined($vars->{PIDDIR})) {
651                 print CF "\tpid directory = $vars->{PIDDIR}\n";
652         }
653         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
654                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
655         }
656         print CF "
657         private dir = $prefix_abs/client/private
658         js include = $srcdir_abs/scripting/libjs
659         name resolve order = bcast
660         interfaces = $interfaces
661         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
662         max xmit = 32K
663         notify:inotify = false
664         ldb:nosync = true
665         system:anonymous = true
666         torture:basedir = $prefix_abs/client
667 #We don't want to pass our self-tests if the PAC code is wrong
668         gensec:require_pac = true
669 ";
670         close(CF);
671 }
672
673
674 my @torture_options = ();
675 push (@torture_options, "--configfile=$conffile");
676 # ensure any one smbtorture call doesn't run too long
677 push (@torture_options, "--maximum-runtime=$torture_maxtime");
678 push (@torture_options, "--target=$opt_target");
679 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
680 push (@torture_options, "--format=subunit");
681 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
682
683 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
684 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
685
686 my @todo = ();
687
688 my $testsdir = "$srcdir/selftest";
689 $ENV{SMB_CONF_PATH} = "$conffile";
690 $ENV{CONFIGURATION} = "--configfile=$conffile";
691
692 my %required_envs = ();
693
694 if ($opt_quick) {
695         open(IN, "$testsdir/tests_quick.sh|");
696 } else {
697         open(IN, "$testsdir/tests_all.sh|");
698 }
699 while (<IN>) {
700         if ($_ eq "-- TEST --\n") {
701                 my $name = <IN>;
702                 $name =~ s/\n//g;
703                 my $env = <IN>;
704                 $env =~ s/\n//g;
705                 my $cmdline = <IN>;
706                 $cmdline =~ s/\n//g;
707                 if (not defined($tests) or $name =~ /$tests/) {
708                         $required_envs{$env} = 1;
709                         push (@todo, [$name, $env, $cmdline]);
710                 }
711         } else {
712                 print;
713         }
714 }
715 close(IN) or die("Error creating recipe");
716
717 my $suitestotal = $#todo + 1;
718 my $i = 0;
719 $| = 1;
720
721 my %running_envs = ();
722
723 my @exported_envvars = (
724         # domain stuff
725         "DOMAIN",
726         "REALM",
727
728         # domain controller stuff
729         "DC_SERVER",
730         "DC_SERVER_IP",
731         "DC_NETBIOSNAME",
732         "DC_NETBIOSALIAS",
733
734         # server stuff
735         "SERVER",
736         "SERVER_IP",
737         "NETBIOSNAME",
738         "NETBIOSALIAS",
739
740         # user stuff
741         "USERNAME",
742         "PASSWORD",
743         "DC_USERNAME",
744         "DC_PASSWORD",
745
746         # misc stuff
747         "KRB5_CONFIG",
748         "WINBINDD_SOCKET_DIR"
749 );
750
751 sub setup_env($)
752 {
753         my ($envname) = @_;
754
755         my $testenv_vars;
756         if ($envname eq "none") {
757                 $testenv_vars = {};
758         } elsif (defined($running_envs{$envname})) {
759                 $testenv_vars = $running_envs{$envname};
760                 if (not $target->check_env($testenv_vars)) {
761                         $testenv_vars = undef;
762                 }
763         } else {
764                 $testenv_vars = $target->setup_env($envname, $prefix);
765         }
766
767         return undef unless defined($testenv_vars);
768
769         SocketWrapper::set_default_iface(6);
770         write_clientconf($conffile, $testenv_vars);
771
772         foreach (@exported_envvars) {
773                 if (defined($testenv_vars->{$_})) {
774                         $ENV{$_} = $testenv_vars->{$_};
775                 } else {
776                         delete $ENV{$_};
777                 }
778         }
779
780         $running_envs{$envname} = $testenv_vars;
781         return $testenv_vars;
782 }
783
784 sub exported_envvars_str($)
785 {
786         my ($testenv_vars) = @_;
787         my $out = "";
788
789         foreach (@exported_envvars) {
790                 next unless defined($testenv_vars->{$_});
791                 $out .= $_."=".$testenv_vars->{$_}."\n";
792         }
793
794         return $out;
795 }
796
797 sub getlog_env($)
798 {
799         my ($envname) = @_;
800         return "" if ($envname eq "none");
801         return $target->getlog_env($running_envs{$envname});
802 }
803
804 sub check_env($)
805 {
806         my ($envname) = @_;
807         return 1 if ($envname eq "none");
808         return $target->check_env($running_envs{$envname});
809 }
810
811 sub teardown_env($)
812 {
813         my ($envname) = @_;
814         return if ($envname eq "none");
815         $target->teardown_env($running_envs{$envname});
816         delete $running_envs{$envname};
817 }
818
819 my $msg_ops;
820 if ($from_build_farm) {
821         $msg_ops = $buildfarm_msg_ops;
822 } else {
823         $msg_ops = $plain_msg_ops;
824 }
825
826 if ($opt_no_lazy_setup) {
827         setup_env($_) foreach (keys %required_envs);
828 }
829
830 if ($opt_testenv) {
831         my $testenv_name = $ENV{SELFTEST_TESTENV};
832         $testenv_name = "dc" unless defined($testenv_name);
833
834         my $testenv_vars = setup_env($testenv_name);
835
836         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
837
838         my $envvarstr = exported_envvars_str($testenv_vars);
839
840         my $term = ($ENV{TERM} or "xterm");
841         system("$term -e 'echo -e \"
842 Welcome to the Samba4 Test environment '$testenv_name'
843
844 This matches the client environment used in make test
845 smbd is pid `cat \$PIDDIR/smbd.pid`
846
847 Some useful environment variables:
848 TORTURE_OPTIONS=\$TORTURE_OPTIONS
849 CONFIGURATION=\$CONFIGURATION
850
851 $envvarstr
852 \" && bash'");
853         teardown_env($testenv_name);
854 } else {
855         foreach (@todo) {
856                 $i++;
857                 my $cmd = $$_[2];
858                 $cmd =~ s/([\(\)])/\\$1/g;
859                 my $name = $$_[0];
860                 my $envname = $$_[1];
861                 
862                 if (skip($name)) {
863                         print "SKIPPED: $name\n";
864                         $statistics->{SUITES_SKIPPED}++;
865                         next;
866                 }
867
868                 my $envvars = setup_env($envname);
869                 if (not defined($envvars)) {
870                         push(@$suitesfailed, $name);
871                         $statistics->{SUITES_FAIL}++;
872                         $statistics->{TESTS_ERROR}++;
873                         print "FAIL: $name (ENV[$envname] not available!)\n";
874                         next;
875                 }
876
877                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
878
879                 if (defined($opt_analyse_cmd)) {
880                         system("$opt_analyse_cmd \"$name\"");
881                 }
882
883                 teardown_env($envname) if ($opt_resetup_env);
884         }
885 }
886
887 print "\n";
888
889 teardown_env($_) foreach (keys %running_envs);
890
891 $target->stop();
892
893 my $end = time();
894 my $duration = ($end-$start);
895 my $numfailed = $#$suitesfailed+1;
896 if ($numfailed == 0) {
897         my $ok = $statistics->{TESTS_EXPECTED_OK} + 
898                  $statistics->{TESTS_EXPECTED_FAIL};
899         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
900 } else {
901         unless ($from_build_farm) {
902                 if (not $opt_immediate and not $opt_verbose) {
903                         foreach (@$suitesfailed) {
904                                 print "===============================================================================\n";
905                                 print "FAIL: $_\n";
906                                 print $test_output->{$_};
907                                 print "\n";
908                         }
909                 }
910
911                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
912         }
913 }
914 print "DURATION: $duration seconds\n";
915
916 my $failed = 0;
917
918 # if there were any valgrind failures, show them
919 foreach (<$prefix/valgrind.log*>) {
920         next unless (-s $_);
921         system("grep DWARF2.CFI.reader $_ > /dev/null");
922         if ($? >> 8 == 0) {
923             print "VALGRIND FAILURE\n";
924             $failed++;
925             system("cat $_");
926         }
927 }
928
929 if ($from_build_farm) {
930         print "TEST STATUS: $numfailed\n";
931 }
932
933 exit $numfailed;