source4/selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use env::Samba3;
 118 use env::Samba4;
 119 use env::Windows;
 120 use SocketWrapper;
 121
 122 my $opt_help = 0;
 123 my $opt_target = "samba4";
 124 my $opt_quick = 0;
 125 my $opt_socket_wrapper = 0;
 126 my $opt_socket_wrapper_pcap = undef;
 127 my $opt_socket_wrapper_keep_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133 my $opt_testenv = 0;
 134 my $ldap = undef;
 135 my $opt_analyse_cmd = undef;
 136 my $opt_resetup_env = undef;
 137 my $opt_bindir = undef;
 138 my $opt_no_lazy_setup = undef;
 139 my $opt_format = "plain";
 140
 141 my $srcdir = ".";
 142 my $builddir = ".";
 143 my $prefix = "./st";
 144
 145 my @expected_failures = ();
 146 my @skips = ();
 147
 148 my $statistics = {
 149         START_TIME => time(),
 150
 151         SUITES_FAIL => 0,
 152         SUITES_OK => 0,
 153         SUITES_SKIPPED => 0,
 154
 155         TESTS_UNEXPECTED_OK => 0,
 156         TESTS_EXPECTED_OK => 0,
 157         TESTS_UNEXPECTED_FAIL => 0,
 158         TESTS_EXPECTED_FAIL => 0,
 159         TESTS_ERROR => 0
 160 };
 161
 162 sub expecting_failure($)
 163 {
 164         my $fullname = shift;
 165
 166         foreach (@expected_failures) {
 167                 return 1 if ($fullname =~ /$_/);
 168         }
 169
 170         return 0;
 171 }
 172
 173 sub skip($)
 174 {
 175         my $fullname = shift;
 176
 177         foreach (@skips) {
 178                 return 1 if ($fullname =~ /$_/);
 179         }
 180
 181         return 0;
 182 }
 183
 184 sub getlog_env($);
 185
 186 sub setup_pcap($)
 187 {
 188         my ($state) = @_;
 189
 190         return unless ($opt_socket_wrapper_pcap);
 191         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 192
 193         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 194         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 195
 196         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 197
 198         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 199 }
 200
 201 sub cleanup_pcap($$$)
 202 {
 203         my ($state, $expected_ret, $ret) = @_;
 204
 205         return unless ($opt_socket_wrapper_pcap);
 206         return if ($opt_socket_wrapper_keep_pcap);
 207         return unless ($expected_ret == $ret);
 208         return unless defined($state->{PCAP_FILE});
 209
 210         unlink($state->{PCAP_FILE});
 211         $state->{PCAP_FILE} = undef;
 212 }
 213
 214 sub parse_subunit_results($$$$)
 215 {
 216         my ($msg_ops, $msg_state, $statistics, $fh) = @_;
 217         my $expected_ret = 1;
 218         my $open_tests = {};
 219
 220         while(<$fh>) {
 221                 $msg_ops->output_msg($msg_state, $_);
 222                 if (/^test: (.+)\n/) {
 223                         $open_tests->{$1} = 1;
 224                         $msg_ops->start_test($msg_state, $1);
 225                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 226                         my $result = $1;
 227                         if ($1 eq "success") {
 228                                 delete $open_tests->{$2};
 229                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
 230                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 231                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 232                                 } else {
 233                                         $statistics->{TESTS_EXPECTED_OK}++;
 234                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 235                                 }
 236                         } elsif ($1 eq "failure") {
 237                                 delete $open_tests->{$2};
 238                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
 239                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 240                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 241                                         $expected_ret = 0;
 242                                 } else {
 243                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 244                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 245                                 }
 246                         } elsif ($1 eq "skip") {
 247                                 delete $open_tests->{$2};
 248                                 $msg_ops->end_test($msg_state, $2, $1, 0);
 249                         } elsif ($1 eq "error") {
 250                                 $statistics->{TESTS_ERROR}++;
 251                                 delete $open_tests->{$2};
 252                                 $msg_ops->end_test($msg_state, $2, $1, 1);
 253                         }
 254                 }
 255         }
 256
 257         foreach (keys %$open_tests) {
 258                 $msg_ops->end_test($msg_state, $_, "error", 1);
 259                 $msg_ops->output_msg($msg_state, "$_ was started but never finished!");
 260                 $statistics->{TESTS_ERROR}++;
 261         }
 262
 263         return $expected_ret;
 264 }
 265
 266 sub run_test($$$$$$)
 267 {
 268         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 269         my $msg_state = {
 270                 ENVNAME => $envname,
 271                 NAME    => $name,
 272                 CMD     => $cmd,
 273                 INDEX   => $i,
 274                 TOTAL   => $totalsuites,
 275                 START_TIME      => time()
 276         };
 277
 278         setup_pcap($msg_state);
 279
 280         open(RESULT, "$cmd 2>&1|");
 281         $msg_ops->start_testsuite($msg_state);
 282
 283         my $expected_ret = parse_subunit_results(
 284                 $msg_ops, $msg_state, $statistics, *RESULT);
 285
 286         my $ret = close(RESULT);
 287
 288         cleanup_pcap($msg_state, $expected_ret, $ret);
 289
 290         $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
 291                                                         getlog_env($msg_state->{ENVNAME}));
 292
 293         if (not $opt_socket_wrapper_keep_pcap and
 294                 defined($msg_state->{PCAP_FILE})) {
 295                 $msg_ops->output_msg($msg_state,
 296                         "PCAP FILE: $msg_state->{PCAP_FILE}\n");
 297         }
 298
 299         if ($ret != $expected_ret) {
 300                 $statistics->{SUITES_FAIL}++;
 301                 exit(1) if ($opt_one);
 302         } else {
 303                 $statistics->{SUITES_OK}++;
 304         }
 305
 306         return ($ret == $expected_ret);
 307 }
 308
 309 sub ShowHelp()
 310 {
 311         print "Samba test runner
 312 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 313
 314 Usage: $Script [OPTIONS] PREFIX
 315
 316 Generic options:
 317  --help                     this help page
 318  --target=samba4|samba3|win Samba version to target
 319
 320 Paths:
 321  --prefix=DIR               prefix to run tests in [st]
 322  --srcdir=DIR               source directory [.]
 323  --builddir=DIR             output directory [.]
 324
 325 Target Specific:
 326  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 327  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 328                             failed
 329  --socket-wrapper           enable socket wrapper
 330  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 331
 332 Samba4 Specific:
 333  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
 334
 335 Samba3 Specific:
 336  --bindir=PATH              path to binaries
 337
 338 Behaviour:
 339  --quick                    run quick overall test
 340  --one                      abort when the first test fails
 341  --immediate                print test output for failed tests during run
 342  --verbose                  be verbose
 343  --analyse-cmd CMD          command to run after each test
 344 ";
 345         exit(0);
 346 }
 347
 348 my $result = GetOptions (
 349                 'help|h|?' => \$opt_help,
 350                 'target=s' => \$opt_target,
 351                 'prefix=s' => \$prefix,
 352                 'socket-wrapper' => \$opt_socket_wrapper,
 353                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 354                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 355                 'quick' => \$opt_quick,
 356                 'one' => \$opt_one,
 357                 'immediate' => \$opt_immediate,
 358                 'expected-failures=s' => \$opt_expected_failures,
 359                 'skip=s' => \$opt_skip,
 360                 'srcdir=s' => \$srcdir,
 361                 'builddir=s' => \$builddir,
 362                 'verbose' => \$opt_verbose,
 363                 'testenv' => \$opt_testenv,
 364                 'ldap:s' => \$ldap,
 365                 'analyse-cmd=s' => \$opt_analyse_cmd,
 366                 'no-lazy-setup' => \$opt_no_lazy_setup,
 367                 'resetup-environment' => \$opt_resetup_env,
 368                 'bindir:s' => \$opt_bindir,
 369                 'format=s' => \$opt_format,
 370             );
 371
 372 exit(1) if (not $result);
 373
 374 ShowHelp() if ($opt_help);
 375
 376 my $tests = shift;
 377
 378 # quick hack to disable rpc validation when using valgrind - its way too slow
 379 unless (defined($ENV{VALGRIND})) {
 380         $ENV{VALIDATE} = "validate";
 381         $ENV{MALLOC_CHECK_} = 2;
 382 }
 383
 384 my $old_pwd = "$RealBin/..";
 385
 386 # Backwards compatibility:
 387 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 388         if (defined($ENV{FEDORA_DS_PREFIX})) {
 389                 $ldap = "fedora-ds";
 390         } else {
 391                 $ldap = "openldap";
 392         }
 393 }
 394
 395 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 396 if ($ldap) {
 397         # LDAP is slow
 398         $torture_maxtime *= 2;
 399 }
 400
 401 $prefix =~ s+//+/+;
 402 $prefix =~ s+/./+/+;
 403 $prefix =~ s+/$++;
 404
 405 die("using an empty prefix isn't allowed") unless $prefix ne "";
 406
 407 #Ensure we have the test prefix around
 408 mkdir($prefix, 0777) unless -d $prefix;
 409
 410 my $prefix_abs = abs_path($prefix);
 411 my $srcdir_abs = abs_path($srcdir);
 412
 413 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 414 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 415
 416 $ENV{PREFIX} = $prefix;
 417 $ENV{PREFIX_ABS} = $prefix_abs;
 418 $ENV{SRCDIR} = $srcdir;
 419 $ENV{SRCDIR_ABS} = $srcdir_abs;
 420
 421 my $tls_enabled = not $opt_quick;
 422 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
 423         ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
 424         $opt_format = "buildfarm";
 425 }
 426
 427 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 428 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 429 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 430 if (defined($ENV{LD_LIBRARY_PATH})) {
 431         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 432 } else {
 433         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 434 }
 435 if (defined($ENV{PKG_CONFIG_PATH})) {
 436         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 437 } else {
 438         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
 439 }
 440 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 441
 442
 443 if ($opt_socket_wrapper_pcap) {
 444         # Socket wrapper pcap implies socket wrapper
 445         $opt_socket_wrapper = 1;
 446 }
 447
 448 my $socket_wrapper_dir;
 449 if ($opt_socket_wrapper) {
 450         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 451         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 452 } else {
 453         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 454 }
 455
 456 my $target;
 457
 458 if ($opt_target eq "samba4") {
 459         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 460 } elsif ($opt_target eq "samba3") {
 461         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 462                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 463         }
 464
 465         $target = new Samba3($opt_bindir);
 466 } elsif ($opt_target eq "win") {
 467         die("Windows tests will not run with socket wrapper enabled.")
 468                 if ($opt_socket_wrapper);
 469         $target = new Windows();
 470 }
 471
 472 if (defined($opt_expected_failures)) {
 473         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 474         while (<KNOWN>) {
 475                 chomp;
 476                 s/([ \t]+)\#(.*)$//;
 477                 push (@expected_failures, $_); }
 478         close(KNOWN);
 479 }
 480
 481 if (defined($opt_skip)) {
 482         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 483         while (<SKIP>) {
 484                 chomp;
 485                 s/([ \t]+)\#(.*)$//;
 486                 push (@skips, $_); }
 487         close(SKIP);
 488 }
 489
 490 my $interfaces = join(',', ("127.0.0.6/8",
 491                             "127.0.0.7/8",
 492                             "127.0.0.8/8",
 493                             "127.0.0.9/8",
 494                             "127.0.0.10/8",
 495                             "127.0.0.11/8"));
 496
 497 my $conffile = "$prefix_abs/client/client.conf";
 498
 499 sub write_clientconf($$)
 500 {
 501         my ($conffile, $vars) = @_;
 502
 503         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 504
 505         if ( -d "$prefix/client/private" ) {
 506                 unlink <$prefix/client/private/*>;
 507         } else {
 508                 mkdir("$prefix/client/private", 0777);
 509         }
 510
 511         open(CF, ">$conffile");
 512         print CF "[global]\n";
 513         if (defined($ENV{VALGRIND})) {
 514                 print CF "\ticonv:native = true\n";
 515         } else {
 516                 print CF "\ticonv:native = false\n";
 517         }
 518         print CF "\tnetbios name = client\n";
 519         if (defined($vars->{DOMAIN})) {
 520                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 521         }
 522         if (defined($vars->{REALM})) {
 523                 print CF "\trealm = $vars->{REALM}\n";
 524         }
 525         if (defined($vars->{NCALRPCDIR})) {
 526                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 527         }
 528         if (defined($vars->{PIDDIR})) {
 529                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 530         }
 531         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 532                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 533         }
 534         print CF "
 535         private dir = $prefix_abs/client/private
 536         js include = $srcdir_abs/scripting/libjs
 537         name resolve order = bcast
 538         interfaces = $interfaces
 539         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 540         max xmit = 32K
 541         notify:inotify = false
 542         ldb:nosync = true
 543         system:anonymous = true
 544         torture:basedir = $prefix_abs/client
 545 #We don't want to pass our self-tests if the PAC code is wrong
 546         gensec:require_pac = true
 547 ";
 548         close(CF);
 549 }
 550
 551
 552 my @torture_options = ();
 553 push (@torture_options, "--configfile=$conffile");
 554 # ensure any one smbtorture call doesn't run too long
 555 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 556 push (@torture_options, "--target=$opt_target");
 557 push (@torture_options, "--option=torture:progress=no") if ($opt_format eq "buildfarm");
 558 push (@torture_options, "--format=subunit");
 559 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 560
 561 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 562 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 563
 564 my @todo = ();
 565
 566 my $testsdir = "$srcdir/selftest";
 567 $ENV{SMB_CONF_PATH} = "$conffile";
 568 $ENV{CONFIGURATION} = "--configfile=$conffile";
 569
 570 my %required_envs = ();
 571
 572 if ($opt_quick) {
 573         open(IN, "$testsdir/tests_quick.sh|");
 574 } else {
 575         open(IN, "$testsdir/tests_all.sh|");
 576 }
 577 while (<IN>) {
 578         if ($_ eq "-- TEST --\n") {
 579                 my $name = <IN>;
 580                 $name =~ s/\n//g;
 581                 my $env = <IN>;
 582                 $env =~ s/\n//g;
 583                 my $cmdline = <IN>;
 584                 $cmdline =~ s/\n//g;
 585                 if (not defined($tests) or $name =~ /$tests/) {
 586                         $required_envs{$env} = 1;
 587                         push (@todo, [$name, $env, $cmdline]);
 588                 }
 589         } else {
 590                 print;
 591         }
 592 }
 593 close(IN) or die("Error creating recipe");
 594
 595 my $suitestotal = $#todo + 1;
 596 my $i = 0;
 597 $| = 1;
 598
 599 my %running_envs = ();
 600
 601 my @exported_envvars = (
 602         # domain stuff
 603         "DOMAIN",
 604         "REALM",
 605
 606         # domain controller stuff
 607         "DC_SERVER",
 608         "DC_SERVER_IP",
 609         "DC_NETBIOSNAME",
 610         "DC_NETBIOSALIAS",
 611
 612         # server stuff
 613         "SERVER",
 614         "SERVER_IP",
 615         "NETBIOSNAME",
 616         "NETBIOSALIAS",
 617
 618         # user stuff
 619         "USERNAME",
 620         "PASSWORD",
 621         "DC_USERNAME",
 622         "DC_PASSWORD",
 623
 624         # misc stuff
 625         "KRB5_CONFIG",
 626         "WINBINDD_SOCKET_DIR"
 627 );
 628
 629 sub setup_env($)
 630 {
 631         my ($envname) = @_;
 632
 633         my $testenv_vars;
 634         if ($envname eq "none") {
 635                 $testenv_vars = {};
 636         } elsif (defined($running_envs{$envname})) {
 637                 $testenv_vars = $running_envs{$envname};
 638                 if (not $target->check_env($testenv_vars)) {
 639                         $testenv_vars = undef;
 640                 }
 641         } else {
 642                 $testenv_vars = $target->setup_env($envname, $prefix);
 643         }
 644
 645         return undef unless defined($testenv_vars);
 646
 647         SocketWrapper::set_default_iface(6);
 648         write_clientconf($conffile, $testenv_vars);
 649
 650         foreach (@exported_envvars) {
 651                 if (defined($testenv_vars->{$_})) {
 652                         $ENV{$_} = $testenv_vars->{$_};
 653                 } else {
 654                         delete $ENV{$_};
 655                 }
 656         }
 657
 658         $running_envs{$envname} = $testenv_vars;
 659         return $testenv_vars;
 660 }
 661
 662 sub exported_envvars_str($)
 663 {
 664         my ($testenv_vars) = @_;
 665         my $out = "";
 666
 667         foreach (@exported_envvars) {
 668                 next unless defined($testenv_vars->{$_});
 669                 $out .= $_."=".$testenv_vars->{$_}."\n";
 670         }
 671
 672         return $out;
 673 }
 674
 675 sub getlog_env($)
 676 {
 677         my ($envname) = @_;
 678         return "" if ($envname eq "none");
 679         return $target->getlog_env($running_envs{$envname});
 680 }
 681
 682 sub check_env($)
 683 {
 684         my ($envname) = @_;
 685         return 1 if ($envname eq "none");
 686         return $target->check_env($running_envs{$envname});
 687 }
 688
 689 sub teardown_env($)
 690 {
 691         my ($envname) = @_;
 692         return if ($envname eq "none");
 693         $target->teardown_env($running_envs{$envname});
 694         delete $running_envs{$envname};
 695 }
 696
 697 my $msg_ops;
 698 if ($opt_format eq "buildfarm") {
 699         require output::buildfarm;
 700         $msg_ops = new output::buildfarm();
 701 } elsif ($opt_format eq "plain") {
 702         require output::plain;
 703         $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
 704 } elsif ($opt_format eq "html") {
 705         require output::html;
 706         mkdir "test-results";
 707         $msg_ops = new output::html("test-results", $statistics);
 708 } else {
 709         die("Invalid output format '$opt_format'");
 710 }
 711
 712 if ($opt_no_lazy_setup) {
 713         setup_env($_) foreach (keys %required_envs);
 714 }
 715
 716 if ($opt_testenv) {
 717         my $testenv_name = $ENV{SELFTEST_TESTENV};
 718         $testenv_name = "dc" unless defined($testenv_name);
 719
 720         my $testenv_vars = setup_env($testenv_name);
 721
 722         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 723
 724         my $envvarstr = exported_envvars_str($testenv_vars);
 725
 726         my $term = ($ENV{TERM} or "xterm");
 727         system("$term -e 'echo -e \"
 728 Welcome to the Samba4 Test environment '$testenv_name'
 729
 730 This matches the client environment used in make test
 731 smbd is pid `cat \$PIDDIR/smbd.pid`
 732
 733 Some useful environment variables:
 734 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 735 CONFIGURATION=\$CONFIGURATION
 736
 737 $envvarstr
 738 \" && bash'");
 739         teardown_env($testenv_name);
 740 } else {
 741         foreach (@todo) {
 742                 $i++;
 743                 my $cmd = $$_[2];
 744                 $cmd =~ s/([\(\)])/\\$1/g;
 745                 my $name = $$_[0];
 746                 my $envname = $$_[1];
 747
 748                 if (skip($name)) {
 749                         $msg_ops->skip_testsuite($name);
 750                         $statistics->{SUITES_SKIPPED}++;
 751                         next;
 752                 }
 753
 754                 my $envvars = setup_env($envname);
 755                 if (not defined($envvars)) {
 756                         $statistics->{SUITES_FAIL}++;
 757                         $statistics->{TESTS_ERROR}++;
 758                         $msg_ops->missing_env($name, $envname);
 759                         next;
 760                 }
 761
 762                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 763
 764                 if (defined($opt_analyse_cmd)) {
 765                         system("$opt_analyse_cmd \"$name\"");
 766                 }
 767
 768                 teardown_env($envname) if ($opt_resetup_env);
 769         }
 770 }
 771
 772 print "\n";
 773
 774 teardown_env($_) foreach (keys %running_envs);
 775
 776 $target->stop();
 777
 778 $statistics->{END_TIME} = time();
 779 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
 780 my $numfailed = $statistics->{SUITES_FAIL};
 781 if ($numfailed == 0) {
 782         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 783                  $statistics->{TESTS_EXPECTED_FAIL};
 784         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 785 } else {
 786         $msg_ops->summary();
 787 }
 788 print "DURATION: $duration seconds\n";
 789
 790 my $failed = 0;
 791
 792 # if there were any valgrind failures, show them
 793 foreach (<$prefix/valgrind.log*>) {
 794         next unless (-s $_);
 795         system("grep DWARF2.CFI.reader $_ > /dev/null");
 796         if ($? >> 8 == 0) {
 797             print "VALGRIND FAILURE\n";
 798             $failed++;
 799             system("cat $_");
 800         }
 801 }
 802
 803 if ($opt_format eq "buildfarm") {
 804         print "TEST STATUS: $numfailed\n";
 805 }
 806
 807 exit $numfailed;