source/selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use Samba3;
 118 use Samba4;
 119 use SocketWrapper;
 120
 121 my $opt_help = 0;
 122 my $opt_target = "samba4";
 123 my $opt_quick = 0;
 124 my $opt_socket_wrapper = 0;
 125 my $opt_socket_wrapper_pcap = undef;
 126 my $opt_socket_wrapper_keep_pcap = undef;
 127 my $opt_one = 0;
 128 my $opt_immediate = 0;
 129 my $opt_expected_failures = undef;
 130 my $opt_skip = undef;
 131 my $opt_verbose = 0;
 132 my $opt_testenv = 0;
 133 my $ldap = undef;
 134 my $opt_analyse_cmd = undef;
 135 my $opt_resetup_env = undef;
 136 my $opt_bindir = undef;
 137 my $opt_no_lazy_setup = undef;
 138
 139 my $srcdir = ".";
 140 my $builddir = ".";
 141 my $prefix = "./st";
 142
 143 my $suitesfailed = [];
 144 my $start = time();
 145 my @expected_failures = ();
 146 my @skips = ();
 147
 148 my $statistics = {
 149         SUITES_FAIL => 0,
 150         SUITES_OK => 0,
 151         SUITES_SKIPPED => 0,
 152
 153         TESTS_UNEXPECTED_OK => 0,
 154         TESTS_EXPECTED_OK => 0,
 155         TESTS_UNEXPECTED_FAIL => 0,
 156         TESTS_EXPECTED_FAIL => 0,
 157         TESTS_ERROR => 0
 158 };
 159
 160 sub expecting_failure($)
 161 {
 162         my $fullname = shift;
 163
 164         foreach (@expected_failures) {
 165                 return 1 if ($fullname =~ /$_/);
 166         }
 167
 168         return 0;
 169 }
 170
 171 sub skip($)
 172 {
 173         my $fullname = shift;
 174
 175         foreach (@skips) {
 176                 return 1 if ($fullname =~ /$_/);
 177         }
 178
 179         return 0;
 180 }
 181
 182 sub getlog_env($);
 183
 184 my $test_output = {};
 185
 186 sub buildfarm_start_msg($)
 187 {
 188         my ($state) = @_;
 189         my $out = "";
 190
 191         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 192         $out .= "Running test $state->{NAME} (level 0 stdout)\n";
 193         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 194         $out .= scalar(localtime())."\n";
 195         $out .= "SELFTEST RUNTIME: " . ($state->{START} - $start) . "s\n";
 196         $out .= "NAME: $state->{NAME}\n";
 197         $out .= "CMD: $state->{CMD}\n";
 198
 199         $test_output->{$state->{NAME}} = "";
 200
 201         print $out;
 202 }
 203
 204 sub buildfarm_output_msg($$)
 205 {
 206         my ($state, $output) = @_;
 207
 208         $test_output->{$state->{NAME}} .= $output;
 209 }
 210
 211 sub buildfarm_end_msg($$$)
 212 {
 213         my ($state, $expected_ret, $ret) = @_;
 214         my $out = "";
 215
 216         $out .= "TEST RUNTIME: " . (time() - $state->{START}) . "s\n";
 217
 218         if ($ret == $expected_ret) {
 219                 $out .= "ALL OK\n";
 220         } else {
 221                 $out .= "ERROR: $ret";
 222                 $out .= $test_output->{$state->{NAME}};
 223         }
 224
 225         $out .= "PCAP FILE: $state->{PCAP_FILE}\n" if defined($state->{PCAP_FILE});
 226
 227         $out .= getlog_env($state->{ENVNAME});
 228
 229         $out .= "==========================================\n";
 230         if ($ret == $expected_ret) {
 231                 $out .= "TEST PASSED: $state->{NAME}\n";
 232         } else {
 233                 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
 234         }
 235         $out .= "==========================================\n";
 236
 237         print $out;
 238 }
 239
 240 my $buildfarm_msg_ops = {
 241         start_msg       => \&buildfarm_start_msg,
 242         output_msg      => \&buildfarm_output_msg,
 243         end_msg         => \&buildfarm_end_msg
 244 };
 245
 246 sub plain_output_msg($$);
 247
 248 sub plain_start_msg($)
 249 {
 250         my ($state) = @_;
 251         my $out = "";
 252
 253         $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
 254         $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
 255         $out .= "] $state->{NAME}\n";
 256
 257         $test_output->{$state->{NAME}} = "" unless $opt_verbose;
 258
 259         plain_output_msg($state, "CMD: $state->{CMD}\n");
 260
 261         print $out;
 262 }
 263
 264 sub plain_output_msg($$)
 265 {
 266         my ($state, $output) = @_;
 267
 268         if ($opt_verbose) {
 269                 print $output;
 270         } else {
 271                 $test_output->{$state->{NAME}} .= $output;
 272         }
 273 }
 274
 275 sub plain_end_msg($$$)
 276 {
 277         my ($state, $expected_ret, $ret) = @_;
 278         my $out = "";
 279
 280         if ($ret != $expected_ret) {
 281                 plain_output_msg($state, "ERROR: $ret\n");
 282         }
 283
 284         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 285                 $out .= $test_output->{$state->{NAME}};
 286         }
 287
 288         if (not $opt_socket_wrapper_keep_pcap and defined($state->{PCAP_FILE})) {
 289                 $out .= "PCAP FILE: $state->{PCAP_FILE}\n";
 290         }
 291
 292         $out .= getlog_env($state->{ENVNAME});
 293
 294         print $out;
 295 }
 296
 297 my $plain_msg_ops = {
 298         start_msg       => \&plain_start_msg,
 299         output_msg      => \&plain_output_msg,
 300         end_msg         => \&plain_end_msg
 301 };
 302
 303 sub setup_pcap($)
 304 {
 305         my ($state) = @_;
 306
 307         return unless ($opt_socket_wrapper_pcap);
 308         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 309
 310         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 311         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 312
 313         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 314
 315         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 316 }
 317
 318 sub cleanup_pcap($$$)
 319 {
 320         my ($state, $expected_ret, $ret) = @_;
 321
 322         return unless ($opt_socket_wrapper_pcap);
 323         return if ($opt_socket_wrapper_keep_pcap);
 324         return unless ($expected_ret == $ret);
 325         return unless defined($state->{PCAP_FILE});
 326
 327         unlink($state->{PCAP_FILE});
 328         $state->{PCAP_FILE} = undef;
 329 }
 330
 331 sub run_test($$$$$$)
 332 {
 333         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 334         my $expected_ret = 1;
 335         my $open_tests = {};
 336         my $msg_state = {
 337                 ENVNAME => $envname,
 338                 NAME    => $name,
 339                 CMD     => $cmd,
 340                 INDEX   => $i,
 341                 TOTAL   => $totalsuites,
 342                 START   => time()
 343         };
 344
 345         setup_pcap($msg_state);
 346
 347         $msg_ops->{start_msg}->($msg_state);
 348
 349         open(RESULT, "$cmd 2>&1|");
 350         while (<RESULT>) {
 351                 $msg_ops->{output_msg}->($msg_state, $_);
 352                 if (/^test: (.+)\n/) {
 353                         $open_tests->{$1} = 1;
 354                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 355                         my $result = $1;
 356                         if ($1 eq "success") {
 357                                 delete $open_tests->{$2};
 358                                 if (expecting_failure("$name/$2")) {
 359                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 360                                 } else {
 361                                         $statistics->{TESTS_EXPECTED_OK}++;
 362                                 }
 363                         } elsif ($1 eq "failure") {
 364                                 delete $open_tests->{$2};
 365                                 if (expecting_failure("$name/$2")) {
 366                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 367                                         $expected_ret = 0;
 368                                 } else {
 369                                         print "n:$name/$2\n";
 370                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 371                                 }
 372                         } elsif ($1 eq "skip") {
 373                                 delete $open_tests->{$2};
 374                         } elsif ($1 eq "error") {
 375                                 $statistics->{TESTS_ERROR}++;
 376                                 delete $open_tests->{$2};
 377                         }
 378                 }
 379         }
 380         foreach (keys %$open_tests) {
 381                 $msg_ops->{output_msg}->($msg_state, "$_ was started but never finished!\n");
 382                 $statistics->{TESTS_ERROR}++;
 383         }
 384         my $ret = close(RESULT);
 385
 386         cleanup_pcap($msg_state,  $expected_ret, $ret);
 387
 388         $msg_ops->{end_msg}->($msg_state, $expected_ret, $ret);
 389
 390         if ($ret != $expected_ret) {
 391                 push(@$suitesfailed, $name);
 392                 $statistics->{SUITES_FAIL}++;
 393                 exit(1) if ($opt_one);
 394         } else {
 395                 $statistics->{SUITES_OK}++;
 396         }
 397
 398         return ($ret == $expected_ret);
 399 }
 400
 401 sub ShowHelp()
 402 {
 403         print "Samba test runner
 404 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 405
 406 Usage: $Script [OPTIONS] PREFIX
 407
 408 Generic options:
 409  --help                     this help page
 410  --target=samba4|samba3|win Samba version to target
 411
 412 Paths:
 413  --prefix=DIR               prefix to run tests in [st]
 414  --srcdir=DIR               source directory [.]
 415  --builddir=DIR             output directory [.]
 416
 417 Target Specific:
 418  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 419  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 420                             failed
 421  --socket-wrapper           enable socket wrapper
 422  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 423
 424 Samba4 Specific:
 425  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
 426
 427 Samba3 Specific:
 428  --bindir=PATH              path to binaries
 429
 430 Behaviour:
 431  --quick                    run quick overall test
 432  --one                      abort when the first test fails
 433  --immediate                print test output for failed tests during run
 434  --verbose                  be verbose
 435  --analyse-cmd CMD          command to run after each test
 436 ";
 437         exit(0);
 438 }
 439
 440 my $result = GetOptions (
 441                 'help|h|?' => \$opt_help,
 442                 'target=s' => \$opt_target,
 443                 'prefix=s' => \$prefix,
 444                 'socket-wrapper' => \$opt_socket_wrapper,
 445                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 446                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 447                 'quick' => \$opt_quick,
 448                 'one' => \$opt_one,
 449                 'immediate' => \$opt_immediate,
 450                 'expected-failures=s' => \$opt_expected_failures,
 451                 'skip=s' => \$opt_skip,
 452                 'srcdir=s' => \$srcdir,
 453                 'builddir=s' => \$builddir,
 454                 'verbose' => \$opt_verbose,
 455                 'testenv' => \$opt_testenv,
 456                 'ldap:s' => \$ldap,
 457                 'analyse-cmd=s' => \$opt_analyse_cmd,
 458                 'no-lazy-setup' => \$opt_no_lazy_setup,
 459                 'resetup-environment' => \$opt_resetup_env,
 460                 'bindir:s' => \$opt_bindir,
 461             );
 462
 463 exit(1) if (not $result);
 464
 465 ShowHelp() if ($opt_help);
 466
 467 my $tests = shift;
 468
 469 # quick hack to disable rpc validation when using valgrind - its way too slow
 470 unless (defined($ENV{VALGRIND})) {
 471         $ENV{VALIDATE} = "validate";
 472         $ENV{MALLOC_CHECK_} = 2;
 473 }
 474
 475 my $old_pwd = "$RealBin/..";
 476
 477 # Backwards compatibility:
 478 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 479         if (defined($ENV{FEDORA_DS_PREFIX})) {
 480                 $ldap = "fedora-ds";
 481         } else {
 482                 $ldap = "openldap";
 483         }
 484 }
 485
 486 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 487 if ($ldap) {
 488         # LDAP is slow
 489         $torture_maxtime *= 2;
 490 }
 491
 492 $prefix =~ s+//+/+;
 493 $prefix =~ s+/./+/+;
 494 $prefix =~ s+/$++;
 495
 496 die("using an empty prefix isn't allowed") unless $prefix ne "";
 497
 498 #Ensure we have the test prefix around
 499 mkdir($prefix, 0777) unless -d $prefix;
 500
 501 my $prefix_abs = abs_path($prefix);
 502 my $srcdir_abs = abs_path($srcdir);
 503
 504 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 505 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 506
 507 $ENV{PREFIX} = $prefix;
 508 $ENV{PREFIX_ABS} = $prefix_abs;
 509 $ENV{SRCDIR} = $srcdir;
 510 $ENV{SRCDIR_ABS} = $srcdir_abs;
 511
 512 my $tls_enabled = not $opt_quick;
 513 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 514                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 515
 516 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 517 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 518 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 519 if (defined($ENV{LD_LIBRARY_PATH})) {
 520         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 521 } else {
 522         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 523 }
 524 if (defined($ENV{PKG_CONFIG_PATH})) {
 525         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 526 } else {
 527         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
 528 }
 529 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 530
 531
 532 if ($opt_socket_wrapper_pcap) {
 533         # Socket wrapper pcap implies socket wrapper
 534         $opt_socket_wrapper = 1;
 535 }
 536
 537 my $socket_wrapper_dir;
 538 if ($opt_socket_wrapper) {
 539         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 540         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 541 } else {
 542         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 543 }
 544
 545 my $target;
 546
 547 if ($opt_target eq "samba4") {
 548         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 549 } elsif ($opt_target eq "samba3") {
 550         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 551                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 552         }
 553
 554         $target = new Samba3($opt_bindir);
 555 } elsif ($opt_target eq "win") {
 556         die("Windows tests will not run with socket wrapper enabled.")
 557                 if ($opt_socket_wrapper);
 558         $target = new Windows();
 559 }
 560
 561 if (defined($opt_expected_failures)) {
 562         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 563         while (<KNOWN>) {
 564                 chomp;
 565                 s/([ \t]+)\#(.*)$//;
 566                 push (@expected_failures, $_); }
 567         close(KNOWN);
 568 }
 569
 570 if (defined($opt_skip)) {
 571         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 572         while (<SKIP>) {
 573                 chomp;
 574                 s/([ \t]+)\#(.*)$//;
 575                 push (@skips, $_); }
 576         close(SKIP);
 577 }
 578
 579 my $interfaces = join(',', ("127.0.0.6/8",
 580                             "127.0.0.7/8",
 581                             "127.0.0.8/8",
 582                             "127.0.0.9/8",
 583                             "127.0.0.10/8",
 584                             "127.0.0.11/8"));
 585
 586 my $conffile = "$prefix_abs/client/client.conf";
 587
 588 sub write_clientconf($$)
 589 {
 590         my ($conffile, $vars) = @_;
 591
 592         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 593
 594         if ( -d "$prefix/client/private" ) {
 595                 unlink <$prefix/client/private/*>;
 596         } else {
 597                 mkdir("$prefix/client/private", 0777);
 598         }
 599
 600         open(CF, ">$conffile");
 601         print CF "[global]\n";
 602         if (defined($ENV{VALGRIND})) {
 603                 print CF "\ticonv:native = true\n";
 604         } else {
 605                 print CF "\ticonv:native = false\n";
 606         }
 607         print CF "\tnetbios name = client\n";
 608         if (defined($vars->{DOMAIN})) {
 609                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 610         }
 611         if (defined($vars->{REALM})) {
 612                 print CF "\trealm = $vars->{REALM}\n";
 613         }
 614         if (defined($vars->{NCALRPCDIR})) {
 615                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 616         }
 617         if (defined($vars->{PIDDIR})) {
 618                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 619         }
 620         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 621                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 622         }
 623         print CF "
 624         private dir = $prefix_abs/client/private
 625         js include = $srcdir_abs/scripting/libjs
 626         name resolve order = bcast
 627         interfaces = $interfaces
 628         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 629         max xmit = 32K
 630         notify:inotify = false
 631         ldb:nosync = true
 632         system:anonymous = true
 633         torture:basedir = $prefix_abs/client
 634 #We don't want to pass our self-tests if the PAC code is wrong
 635         gensec:require_pac = true
 636 ";
 637         close(CF);
 638 }
 639
 640
 641 my @torture_options = ();
 642 push (@torture_options, "--configfile=$conffile");
 643 # ensure any one smbtorture call doesn't run too long
 644 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 645 push (@torture_options, "--target=$opt_target");
 646 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 647 push (@torture_options, "--format=subunit");
 648 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 649
 650 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 651 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 652
 653 my @todo = ();
 654
 655 my $testsdir = "$srcdir/selftest";
 656 $ENV{SMB_CONF_PATH} = "$conffile";
 657 $ENV{CONFIGURATION} = "--configfile=$conffile";
 658
 659 my %required_envs = ();
 660
 661 if ($opt_quick) {
 662         open(IN, "$testsdir/tests_quick.sh|");
 663 } else {
 664         open(IN, "$testsdir/tests_all.sh|");
 665 }
 666 while (<IN>) {
 667         if ($_ eq "-- TEST --\n") {
 668                 my $name = <IN>;
 669                 $name =~ s/\n//g;
 670                 my $env = <IN>;
 671                 $env =~ s/\n//g;
 672                 my $cmdline = <IN>;
 673                 $cmdline =~ s/\n//g;
 674                 if (not defined($tests) or $name =~ /$tests/) {
 675                         $required_envs{$env} = 1;
 676                         push (@todo, [$name, $env, $cmdline]);
 677                 }
 678         } else {
 679                 print;
 680         }
 681 }
 682 close(IN) or die("Error creating recipe");
 683
 684 my $suitestotal = $#todo + 1;
 685 my $i = 0;
 686 $| = 1;
 687
 688 my %running_envs = ();
 689
 690 my @exported_envvars = (
 691         # domain stuff
 692         "DOMAIN",
 693         "REALM",
 694
 695         # domain controller stuff
 696         "DC_SERVER",
 697         "DC_SERVER_IP",
 698         "DC_NETBIOSNAME",
 699         "DC_NETBIOSALIAS",
 700
 701         # server stuff
 702         "SERVER",
 703         "SERVER_IP",
 704         "NETBIOSNAME",
 705         "NETBIOSALIAS",
 706
 707         # user stuff
 708         "USERNAME",
 709         "PASSWORD",
 710         "DC_USERNAME",
 711         "DC_PASSWORD",
 712
 713         # misc stuff
 714         "KRB5_CONFIG",
 715         "WINBINDD_SOCKET_DIR"
 716 );
 717
 718 sub setup_env($)
 719 {
 720         my ($envname) = @_;
 721
 722         my $testenv_vars;
 723         if ($envname eq "none") {
 724                 $testenv_vars = {};
 725         } elsif (defined($running_envs{$envname})) {
 726                 $testenv_vars = $running_envs{$envname};
 727                 if (not $target->check_env($testenv_vars)) {
 728                         $testenv_vars = undef;
 729                 }
 730         } else {
 731                 $testenv_vars = $target->setup_env($envname, $prefix);
 732         }
 733
 734         return undef unless defined($testenv_vars);
 735
 736         SocketWrapper::set_default_iface(6);
 737         write_clientconf($conffile, $testenv_vars);
 738
 739         foreach (@exported_envvars) {
 740                 if (defined($testenv_vars->{$_})) {
 741                         $ENV{$_} = $testenv_vars->{$_};
 742                 } else {
 743                         delete $ENV{$_};
 744                 }
 745         }
 746
 747         $running_envs{$envname} = $testenv_vars;
 748         return $testenv_vars;
 749 }
 750
 751 sub exported_envvars_str($)
 752 {
 753         my ($testenv_vars) = @_;
 754         my $out = "";
 755
 756         foreach (@exported_envvars) {
 757                 next unless defined($testenv_vars->{$_});
 758                 $out .= $_."=".$testenv_vars->{$_}."\n";
 759         }
 760
 761         return $out;
 762 }
 763
 764 sub getlog_env($)
 765 {
 766         my ($envname) = @_;
 767         return "" if ($envname eq "none");
 768         return $target->getlog_env($running_envs{$envname});
 769 }
 770
 771 sub check_env($)
 772 {
 773         my ($envname) = @_;
 774         return 1 if ($envname eq "none");
 775         return $target->check_env($running_envs{$envname});
 776 }
 777
 778 sub teardown_env($)
 779 {
 780         my ($envname) = @_;
 781         return if ($envname eq "none");
 782         $target->teardown_env($running_envs{$envname});
 783         delete $running_envs{$envname};
 784 }
 785
 786 my $msg_ops;
 787 if ($from_build_farm) {
 788         $msg_ops = $buildfarm_msg_ops;
 789 } else {
 790         $msg_ops = $plain_msg_ops;
 791 }
 792
 793 if ($opt_no_lazy_setup) {
 794         setup_env($_) foreach (keys %required_envs);
 795 }
 796
 797 if ($opt_testenv) {
 798         my $testenv_name = $ENV{SELFTEST_TESTENV};
 799         $testenv_name = "dc" unless defined($testenv_name);
 800
 801         my $testenv_vars = setup_env($testenv_name);
 802
 803         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 804
 805         my $envvarstr = exported_envvars_str($testenv_vars);
 806
 807         my $term = ($ENV{TERM} or "xterm");
 808         system("$term -e 'echo -e \"
 809 Welcome to the Samba4 Test environment '$testenv_name'
 810
 811 This matches the client environment used in make test
 812 smbd is pid `cat \$PIDDIR/smbd.pid`
 813
 814 Some useful environment variables:
 815 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 816 CONFIGURATION=\$CONFIGURATION
 817
 818 $envvarstr
 819 \" && bash'");
 820         teardown_env($testenv_name);
 821 } else {
 822         foreach (@todo) {
 823                 $i++;
 824                 my $cmd = $$_[2];
 825                 $cmd =~ s/([\(\)])/\\$1/g;
 826                 my $name = $$_[0];
 827                 my $envname = $$_[1];
 828
 829                 if (skip($name)) {
 830                         print "SKIPPED: $name\n";
 831                         $statistics->{SUITES_SKIPPED}++;
 832                         next;
 833                 }
 834
 835                 my $envvars = setup_env($envname);
 836                 if (not defined($envvars)) {
 837                         push(@$suitesfailed, $name);
 838                         $statistics->{SUITES_FAIL}++;
 839                         $statistics->{TESTS_ERROR}++;
 840                         print "FAIL: $name (ENV[$envname] not available!)\n";
 841                         next;
 842                 }
 843
 844                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 845
 846                 if (defined($opt_analyse_cmd)) {
 847                         system("$opt_analyse_cmd \"$name\"");
 848                 }
 849
 850                 teardown_env($envname) if ($opt_resetup_env);
 851         }
 852 }
 853
 854 print "\n";
 855
 856 teardown_env($_) foreach (keys %running_envs);
 857
 858 $target->stop();
 859
 860 my $end = time();
 861 my $duration = ($end-$start);
 862 my $numfailed = $#$suitesfailed+1;
 863 if ($numfailed == 0) {
 864         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 865                  $statistics->{TESTS_EXPECTED_FAIL};
 866         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 867 } else {
 868         unless ($from_build_farm) {
 869                 if (not $opt_immediate and not $opt_verbose) {
 870                         foreach (@$suitesfailed) {
 871                                 print "===============================================================================\n";
 872                                 print "FAIL: $_\n";
 873                                 print $test_output->{$_};
 874                                 print "\n";
 875                         }
 876                 }
 877
 878                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 879         }
 880 }
 881 print "DURATION: $duration seconds\n";
 882
 883 my $failed = 0;
 884
 885 # if there were any valgrind failures, show them
 886 foreach (<$prefix/valgrind.log*>) {
 887         next unless (-s $_);
 888         system("grep DWARF2.CFI.reader $_ > /dev/null");
 889         if ($? >> 8 == 0) {
 890             print "VALGRIND FAILURE\n";
 891             $failed++;
 892             system("cat $_");
 893         }
 894 }
 895
 896 if ($from_build_farm) {
 897         print "TEST STATUS: $numfailed\n";
 898 }
 899
 900 exit $numfailed;