source4/selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use env::Samba3;
 118 use env::Samba4;
 119 use env::Windows;
 120 use SocketWrapper;
 121
 122 my $opt_help = 0;
 123 my $opt_target = "samba4";
 124 my $opt_quick = 0;
 125 my $opt_socket_wrapper = 0;
 126 my $opt_socket_wrapper_pcap = undef;
 127 my $opt_socket_wrapper_keep_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133 my $opt_testenv = 0;
 134 my $ldap = undef;
 135 my $opt_analyse_cmd = undef;
 136 my $opt_resetup_env = undef;
 137 my $opt_bindir = undef;
 138 my $opt_no_lazy_setup = undef;
 139
 140 my $srcdir = ".";
 141 my $builddir = ".";
 142 my $prefix = "./st";
 143
 144 my @expected_failures = ();
 145 my @skips = ();
 146
 147 my $statistics = {
 148         START_TIME => time(),
 149
 150         SUITES_FAIL => 0,
 151         SUITES_OK => 0,
 152         SUITES_SKIPPED => 0,
 153
 154         TESTS_UNEXPECTED_OK => 0,
 155         TESTS_EXPECTED_OK => 0,
 156         TESTS_UNEXPECTED_FAIL => 0,
 157         TESTS_EXPECTED_FAIL => 0,
 158         TESTS_ERROR => 0
 159 };
 160
 161 sub expecting_failure($)
 162 {
 163         my $fullname = shift;
 164
 165         foreach (@expected_failures) {
 166                 return 1 if ($fullname =~ /$_/);
 167         }
 168
 169         return 0;
 170 }
 171
 172 sub skip($)
 173 {
 174         my $fullname = shift;
 175
 176         foreach (@skips) {
 177                 return 1 if ($fullname =~ /$_/);
 178         }
 179
 180         return 0;
 181 }
 182
 183 sub getlog_env($);
 184
 185 sub setup_pcap($)
 186 {
 187         my ($state) = @_;
 188
 189         return unless ($opt_socket_wrapper_pcap);
 190         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 191
 192         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 193         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 194
 195         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 196
 197         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 198 }
 199
 200 sub cleanup_pcap($$$)
 201 {
 202         my ($state, $expected_ret, $ret) = @_;
 203
 204         return unless ($opt_socket_wrapper_pcap);
 205         return if ($opt_socket_wrapper_keep_pcap);
 206         return unless ($expected_ret == $ret);
 207         return unless defined($state->{PCAP_FILE});
 208
 209         unlink($state->{PCAP_FILE});
 210         $state->{PCAP_FILE} = undef;
 211 }
 212
 213 sub parse_subunit_results($$$$)
 214 {
 215         my ($msg_ops, $msg_state, $statistics, $fh) = @_;
 216         my $expected_ret = 1;
 217         my $open_tests = {};
 218
 219         while(<$fh>) {
 220                 $msg_ops->output_msg($msg_state, $_);
 221                 if (/^test: (.+)\n/) {
 222                         $open_tests->{$1} = 1;
 223                         $msg_ops->start_test($msg_state, $1);
 224                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 225                         my $result = $1;
 226                         if ($1 eq "success") {
 227                                 delete $open_tests->{$2};
 228                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
 229                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 230                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 231                                 } else {
 232                                         $statistics->{TESTS_EXPECTED_OK}++;
 233                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 234                                 }
 235                         } elsif ($1 eq "failure") {
 236                                 delete $open_tests->{$2};
 237                                 if (expecting_failure("$msg_state->{NAME}/$2")) {
 238                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 239                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 240                                         $expected_ret = 0;
 241                                 } else {
 242                                         print "n:$msg_state->{NAME}/$2\n";
 243                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 244                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 245                                 }
 246                         } elsif ($1 eq "skip") {
 247                                 delete $open_tests->{$2};
 248                                 $msg_ops->end_test($msg_state, $2, $1, 0);
 249                         } elsif ($1 eq "error") {
 250                                 $statistics->{TESTS_ERROR}++;
 251                                 delete $open_tests->{$2};
 252                                 $msg_ops->end_test($msg_state, $2, $1, 1);
 253                         }
 254                 }
 255         }
 256
 257         foreach (keys %$open_tests) {
 258                 $msg_ops->end_test($msg_state, $_, "error", 1);
 259                 $msg_ops->output_msg($msg_state, "$_ was started but never finished!");
 260                 $statistics->{TESTS_ERROR}++;
 261         }
 262
 263         return $expected_ret;
 264 }
 265
 266 sub run_test($$$$$$)
 267 {
 268         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 269         my $msg_state = {
 270                 ENVNAME => $envname,
 271                 NAME    => $name,
 272                 CMD     => $cmd,
 273                 INDEX   => $i,
 274                 TOTAL   => $totalsuites,
 275                 START_TIME      => time()
 276         };
 277
 278         setup_pcap($msg_state);
 279
 280         open(RESULT, "$cmd 2>&1|");
 281         $msg_ops->start_testsuite($msg_state);
 282
 283         my $expected_ret = parse_subunit_results(
 284                 $msg_ops, $msg_state, $statistics, *RESULT);
 285
 286         my $ret = close(RESULT);
 287
 288         cleanup_pcap($msg_state,  $expected_ret, $ret);
 289
 290         $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
 291                                                         getlog_env($msg_state->{ENVNAME}));
 292
 293         if (not $opt_socket_wrapper_keep_pcap and
 294                 defined($msg_state->{PCAP_FILE})) {
 295                 $msg_ops->output_msg($msg_state,
 296                         "PCAP FILE: $msg_state->{PCAP_FILE}\n");
 297         }
 298
 299         if ($ret != $expected_ret) {
 300                 $statistics->{SUITES_FAIL}++;
 301                 exit(1) if ($opt_one);
 302         } else {
 303                 $statistics->{SUITES_OK}++;
 304         }
 305
 306         return ($ret == $expected_ret);
 307 }
 308
 309 sub ShowHelp()
 310 {
 311         print "Samba test runner
 312 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 313
 314 Usage: $Script [OPTIONS] PREFIX
 315
 316 Generic options:
 317  --help                     this help page
 318  --target=samba4|samba3|win Samba version to target
 319
 320 Paths:
 321  --prefix=DIR               prefix to run tests in [st]
 322  --srcdir=DIR               source directory [.]
 323  --builddir=DIR             output directory [.]
 324
 325 Target Specific:
 326  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 327  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 328                             failed
 329  --socket-wrapper           enable socket wrapper
 330  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 331
 332 Samba4 Specific:
 333  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
 334
 335 Samba3 Specific:
 336  --bindir=PATH              path to binaries
 337
 338 Behaviour:
 339  --quick                    run quick overall test
 340  --one                      abort when the first test fails
 341  --immediate                print test output for failed tests during run
 342  --verbose                  be verbose
 343  --analyse-cmd CMD          command to run after each test
 344 ";
 345         exit(0);
 346 }
 347
 348 my $result = GetOptions (
 349                 'help|h|?' => \$opt_help,
 350                 'target=s' => \$opt_target,
 351                 'prefix=s' => \$prefix,
 352                 'socket-wrapper' => \$opt_socket_wrapper,
 353                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 354                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 355                 'quick' => \$opt_quick,
 356                 'one' => \$opt_one,
 357                 'immediate' => \$opt_immediate,
 358                 'expected-failures=s' => \$opt_expected_failures,
 359                 'skip=s' => \$opt_skip,
 360                 'srcdir=s' => \$srcdir,
 361                 'builddir=s' => \$builddir,
 362                 'verbose' => \$opt_verbose,
 363                 'testenv' => \$opt_testenv,
 364                 'ldap:s' => \$ldap,
 365                 'analyse-cmd=s' => \$opt_analyse_cmd,
 366                 'no-lazy-setup' => \$opt_no_lazy_setup,
 367                 'resetup-environment' => \$opt_resetup_env,
 368                 'bindir:s' => \$opt_bindir,
 369             );
 370
 371 exit(1) if (not $result);
 372
 373 ShowHelp() if ($opt_help);
 374
 375 my $tests = shift;
 376
 377 # quick hack to disable rpc validation when using valgrind - its way too slow
 378 unless (defined($ENV{VALGRIND})) {
 379         $ENV{VALIDATE} = "validate";
 380         $ENV{MALLOC_CHECK_} = 2;
 381 }
 382
 383 my $old_pwd = "$RealBin/..";
 384
 385 # Backwards compatibility:
 386 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 387         if (defined($ENV{FEDORA_DS_PREFIX})) {
 388                 $ldap = "fedora-ds";
 389         } else {
 390                 $ldap = "openldap";
 391         }
 392 }
 393
 394 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 395 if ($ldap) {
 396         # LDAP is slow
 397         $torture_maxtime *= 2;
 398 }
 399
 400 $prefix =~ s+//+/+;
 401 $prefix =~ s+/./+/+;
 402 $prefix =~ s+/$++;
 403
 404 die("using an empty prefix isn't allowed") unless $prefix ne "";
 405
 406 #Ensure we have the test prefix around
 407 mkdir($prefix, 0777) unless -d $prefix;
 408
 409 my $prefix_abs = abs_path($prefix);
 410 my $srcdir_abs = abs_path($srcdir);
 411
 412 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 413 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 414
 415 $ENV{PREFIX} = $prefix;
 416 $ENV{PREFIX_ABS} = $prefix_abs;
 417 $ENV{SRCDIR} = $srcdir;
 418 $ENV{SRCDIR_ABS} = $srcdir_abs;
 419
 420 my $tls_enabled = not $opt_quick;
 421 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 422                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 423
 424 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 425 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 426 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 427 if (defined($ENV{LD_LIBRARY_PATH})) {
 428         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 429 } else {
 430         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 431 }
 432 if (defined($ENV{PKG_CONFIG_PATH})) {
 433         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 434 } else {
 435         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
 436 }
 437 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 438
 439
 440 if ($opt_socket_wrapper_pcap) {
 441         # Socket wrapper pcap implies socket wrapper
 442         $opt_socket_wrapper = 1;
 443 }
 444
 445 my $socket_wrapper_dir;
 446 if ($opt_socket_wrapper) {
 447         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 448         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 449 } else {
 450         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 451 }
 452
 453 my $target;
 454
 455 if ($opt_target eq "samba4") {
 456         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 457 } elsif ($opt_target eq "samba3") {
 458         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 459                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 460         }
 461
 462         $target = new Samba3($opt_bindir);
 463 } elsif ($opt_target eq "win") {
 464         die("Windows tests will not run with socket wrapper enabled.")
 465                 if ($opt_socket_wrapper);
 466         $target = new Windows();
 467 }
 468
 469 if (defined($opt_expected_failures)) {
 470         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 471         while (<KNOWN>) {
 472                 chomp;
 473                 s/([ \t]+)\#(.*)$//;
 474                 push (@expected_failures, $_); }
 475         close(KNOWN);
 476 }
 477
 478 if (defined($opt_skip)) {
 479         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 480         while (<SKIP>) {
 481                 chomp;
 482                 s/([ \t]+)\#(.*)$//;
 483                 push (@skips, $_); }
 484         close(SKIP);
 485 }
 486
 487 my $interfaces = join(',', ("127.0.0.6/8",
 488                             "127.0.0.7/8",
 489                             "127.0.0.8/8",
 490                             "127.0.0.9/8",
 491                             "127.0.0.10/8",
 492                             "127.0.0.11/8"));
 493
 494 my $conffile = "$prefix_abs/client/client.conf";
 495
 496 sub write_clientconf($$)
 497 {
 498         my ($conffile, $vars) = @_;
 499
 500         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 501
 502         if ( -d "$prefix/client/private" ) {
 503                 unlink <$prefix/client/private/*>;
 504         } else {
 505                 mkdir("$prefix/client/private", 0777);
 506         }
 507
 508         open(CF, ">$conffile");
 509         print CF "[global]\n";
 510         if (defined($ENV{VALGRIND})) {
 511                 print CF "\ticonv:native = true\n";
 512         } else {
 513                 print CF "\ticonv:native = false\n";
 514         }
 515         print CF "\tnetbios name = client\n";
 516         if (defined($vars->{DOMAIN})) {
 517                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 518         }
 519         if (defined($vars->{REALM})) {
 520                 print CF "\trealm = $vars->{REALM}\n";
 521         }
 522         if (defined($vars->{NCALRPCDIR})) {
 523                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 524         }
 525         if (defined($vars->{PIDDIR})) {
 526                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 527         }
 528         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 529                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 530         }
 531         print CF "
 532         private dir = $prefix_abs/client/private
 533         js include = $srcdir_abs/scripting/libjs
 534         name resolve order = bcast
 535         interfaces = $interfaces
 536         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 537         max xmit = 32K
 538         notify:inotify = false
 539         ldb:nosync = true
 540         system:anonymous = true
 541         torture:basedir = $prefix_abs/client
 542 #We don't want to pass our self-tests if the PAC code is wrong
 543         gensec:require_pac = true
 544 ";
 545         close(CF);
 546 }
 547
 548
 549 my @torture_options = ();
 550 push (@torture_options, "--configfile=$conffile");
 551 # ensure any one smbtorture call doesn't run too long
 552 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 553 push (@torture_options, "--target=$opt_target");
 554 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 555 push (@torture_options, "--format=subunit");
 556 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 557
 558 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 559 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 560
 561 my @todo = ();
 562
 563 my $testsdir = "$srcdir/selftest";
 564 $ENV{SMB_CONF_PATH} = "$conffile";
 565 $ENV{CONFIGURATION} = "--configfile=$conffile";
 566
 567 my %required_envs = ();
 568
 569 if ($opt_quick) {
 570         open(IN, "$testsdir/tests_quick.sh|");
 571 } else {
 572         open(IN, "$testsdir/tests_all.sh|");
 573 }
 574 while (<IN>) {
 575         if ($_ eq "-- TEST --\n") {
 576                 my $name = <IN>;
 577                 $name =~ s/\n//g;
 578                 my $env = <IN>;
 579                 $env =~ s/\n//g;
 580                 my $cmdline = <IN>;
 581                 $cmdline =~ s/\n//g;
 582                 if (not defined($tests) or $name =~ /$tests/) {
 583                         $required_envs{$env} = 1;
 584                         push (@todo, [$name, $env, $cmdline]);
 585                 }
 586         } else {
 587                 print;
 588         }
 589 }
 590 close(IN) or die("Error creating recipe");
 591
 592 my $suitestotal = $#todo + 1;
 593 my $i = 0;
 594 $| = 1;
 595
 596 my %running_envs = ();
 597
 598 my @exported_envvars = (
 599         # domain stuff
 600         "DOMAIN",
 601         "REALM",
 602
 603         # domain controller stuff
 604         "DC_SERVER",
 605         "DC_SERVER_IP",
 606         "DC_NETBIOSNAME",
 607         "DC_NETBIOSALIAS",
 608
 609         # server stuff
 610         "SERVER",
 611         "SERVER_IP",
 612         "NETBIOSNAME",
 613         "NETBIOSALIAS",
 614
 615         # user stuff
 616         "USERNAME",
 617         "PASSWORD",
 618         "DC_USERNAME",
 619         "DC_PASSWORD",
 620
 621         # misc stuff
 622         "KRB5_CONFIG",
 623         "WINBINDD_SOCKET_DIR"
 624 );
 625
 626 sub setup_env($)
 627 {
 628         my ($envname) = @_;
 629
 630         my $testenv_vars;
 631         if ($envname eq "none") {
 632                 $testenv_vars = {};
 633         } elsif (defined($running_envs{$envname})) {
 634                 $testenv_vars = $running_envs{$envname};
 635                 if (not $target->check_env($testenv_vars)) {
 636                         $testenv_vars = undef;
 637                 }
 638         } else {
 639                 $testenv_vars = $target->setup_env($envname, $prefix);
 640         }
 641
 642         return undef unless defined($testenv_vars);
 643
 644         SocketWrapper::set_default_iface(6);
 645         write_clientconf($conffile, $testenv_vars);
 646
 647         foreach (@exported_envvars) {
 648                 if (defined($testenv_vars->{$_})) {
 649                         $ENV{$_} = $testenv_vars->{$_};
 650                 } else {
 651                         delete $ENV{$_};
 652                 }
 653         }
 654
 655         $running_envs{$envname} = $testenv_vars;
 656         return $testenv_vars;
 657 }
 658
 659 sub exported_envvars_str($)
 660 {
 661         my ($testenv_vars) = @_;
 662         my $out = "";
 663
 664         foreach (@exported_envvars) {
 665                 next unless defined($testenv_vars->{$_});
 666                 $out .= $_."=".$testenv_vars->{$_}."\n";
 667         }
 668
 669         return $out;
 670 }
 671
 672 sub getlog_env($)
 673 {
 674         my ($envname) = @_;
 675         return "" if ($envname eq "none");
 676         return $target->getlog_env($running_envs{$envname});
 677 }
 678
 679 sub check_env($)
 680 {
 681         my ($envname) = @_;
 682         return 1 if ($envname eq "none");
 683         return $target->check_env($running_envs{$envname});
 684 }
 685
 686 sub teardown_env($)
 687 {
 688         my ($envname) = @_;
 689         return if ($envname eq "none");
 690         $target->teardown_env($running_envs{$envname});
 691         delete $running_envs{$envname};
 692 }
 693
 694 my $msg_ops;
 695 if ($from_build_farm) {
 696         require output::buildfarm;
 697         $msg_ops = new output::buildfarm();
 698 } else {
 699         require output::plain;
 700         $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
 701 }
 702
 703 if ($opt_no_lazy_setup) {
 704         setup_env($_) foreach (keys %required_envs);
 705 }
 706
 707 if ($opt_testenv) {
 708         my $testenv_name = $ENV{SELFTEST_TESTENV};
 709         $testenv_name = "dc" unless defined($testenv_name);
 710
 711         my $testenv_vars = setup_env($testenv_name);
 712
 713         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 714
 715         my $envvarstr = exported_envvars_str($testenv_vars);
 716
 717         my $term = ($ENV{TERM} or "xterm");
 718         system("$term -e 'echo -e \"
 719 Welcome to the Samba4 Test environment '$testenv_name'
 720
 721 This matches the client environment used in make test
 722 smbd is pid `cat \$PIDDIR/smbd.pid`
 723
 724 Some useful environment variables:
 725 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 726 CONFIGURATION=\$CONFIGURATION
 727
 728 $envvarstr
 729 \" && bash'");
 730         teardown_env($testenv_name);
 731 } else {
 732         foreach (@todo) {
 733                 $i++;
 734                 my $cmd = $$_[2];
 735                 $cmd =~ s/([\(\)])/\\$1/g;
 736                 my $name = $$_[0];
 737                 my $envname = $$_[1];
 738
 739                 if (skip($name)) {
 740                         print "SKIPPED: $name\n";
 741                         $statistics->{SUITES_SKIPPED}++;
 742                         next;
 743                 }
 744
 745                 my $envvars = setup_env($envname);
 746                 if (not defined($envvars)) {
 747                         $statistics->{SUITES_FAIL}++;
 748                         $statistics->{TESTS_ERROR}++;
 749                         $msg_ops->missing_env($name, $envname);
 750                         next;
 751                 }
 752
 753                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 754
 755                 if (defined($opt_analyse_cmd)) {
 756                         system("$opt_analyse_cmd \"$name\"");
 757                 }
 758
 759                 teardown_env($envname) if ($opt_resetup_env);
 760         }
 761 }
 762
 763 print "\n";
 764
 765 teardown_env($_) foreach (keys %running_envs);
 766
 767 $target->stop();
 768
 769 $statistics->{END_TIME} = time();
 770 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
 771 my $numfailed = $statistics->{SUITES_FAIL};
 772 if ($numfailed == 0) {
 773         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 774                  $statistics->{TESTS_EXPECTED_FAIL};
 775         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 776 } else {
 777         $msg_ops->summary();
 778 }
 779 print "DURATION: $duration seconds\n";
 780
 781 my $failed = 0;
 782
 783 # if there were any valgrind failures, show them
 784 foreach (<$prefix/valgrind.log*>) {
 785         next unless (-s $_);
 786         system("grep DWARF2.CFI.reader $_ > /dev/null");
 787         if ($? >> 8 == 0) {
 788             print "VALGRIND FAILURE\n";
 789             $failed++;
 790             system("cat $_");
 791         }
 792 }
 793
 794 if ($from_build_farm) {
 795         print "TEST STATUS: $numfailed\n";
 796 }
 797
 798 exit $numfailed;