source4/selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use Samba3;
 118 use Samba4;
 119 use SocketWrapper;
 120
 121 my $opt_help = 0;
 122 my $opt_target = "samba4";
 123 my $opt_quick = 0;
 124 my $opt_socket_wrapper = 0;
 125 my $opt_socket_wrapper_pcap = undef;
 126 my $opt_socket_wrapper_keep_pcap = undef;
 127 my $opt_one = 0;
 128 my $opt_immediate = 0;
 129 my $opt_expected_failures = undef;
 130 my $opt_skip = undef;
 131 my $opt_verbose = 0;
 132 my $opt_testenv = 0;
 133 my $ldap = undef;
 134 my $opt_analyse_cmd = undef;
 135 my $opt_resetup_env = undef;
 136 my $opt_bindir = undef;
 137 my $opt_no_lazy_setup = undef;
 138
 139 my $srcdir = ".";
 140 my $builddir = ".";
 141 my $prefix = "./st";
 142
 143 my @expected_failures = ();
 144 my @skips = ();
 145
 146 my $statistics = {
 147         START_TIME => time(),
 148
 149         SUITES_FAIL => 0,
 150         SUITES_OK => 0,
 151         SUITES_SKIPPED => 0,
 152
 153         TESTS_UNEXPECTED_OK => 0,
 154         TESTS_EXPECTED_OK => 0,
 155         TESTS_UNEXPECTED_FAIL => 0,
 156         TESTS_EXPECTED_FAIL => 0,
 157         TESTS_ERROR => 0
 158 };
 159
 160 sub expecting_failure($)
 161 {
 162         my $fullname = shift;
 163
 164         foreach (@expected_failures) {
 165                 return 1 if ($fullname =~ /$_/);
 166         }
 167
 168         return 0;
 169 }
 170
 171 sub skip($)
 172 {
 173         my $fullname = shift;
 174
 175         foreach (@skips) {
 176                 return 1 if ($fullname =~ /$_/);
 177         }
 178
 179         return 0;
 180 }
 181
 182 sub getlog_env($);
 183
 184 sub setup_pcap($)
 185 {
 186         my ($state) = @_;
 187
 188         return unless ($opt_socket_wrapper_pcap);
 189         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 190
 191         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 192         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 193
 194         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 195
 196         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 197 }
 198
 199 sub cleanup_pcap($$$)
 200 {
 201         my ($state, $expected_ret, $ret) = @_;
 202
 203         return unless ($opt_socket_wrapper_pcap);
 204         return if ($opt_socket_wrapper_keep_pcap);
 205         return unless ($expected_ret == $ret);
 206         return unless defined($state->{PCAP_FILE});
 207
 208         unlink($state->{PCAP_FILE});
 209         $state->{PCAP_FILE} = undef;
 210 }
 211
 212 sub run_test($$$$$$)
 213 {
 214         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 215         my $expected_ret = 1;
 216         my $open_tests = {};
 217         my $msg_state = {
 218                 ENVNAME => $envname,
 219                 NAME    => $name,
 220                 CMD     => $cmd,
 221                 INDEX   => $i,
 222                 TOTAL   => $totalsuites,
 223                 START_TIME      => time()
 224         };
 225
 226         setup_pcap($msg_state);
 227
 228         $msg_ops->start_testsuite($msg_state);
 229
 230         open(RESULT, "$cmd 2>&1|");
 231         while (<RESULT>) {
 232                 $msg_ops->output_msg($msg_state, $_);
 233                 if (/^test: (.+)\n/) {
 234                         $open_tests->{$1} = 1;
 235                         $msg_ops->start_test($msg_state, $1);
 236                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 237                         my $result = $1;
 238                         if ($1 eq "success") {
 239                                 delete $open_tests->{$2};
 240                                 if (expecting_failure("$name/$2")) {
 241                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 242                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 243                                 } else {
 244                                         $statistics->{TESTS_EXPECTED_OK}++;
 245                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 246                                 }
 247                         } elsif ($1 eq "failure") {
 248                                 delete $open_tests->{$2};
 249                                 if (expecting_failure("$name/$2")) {
 250                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 251                                         $msg_ops->end_test($msg_state, $2, $1, 0);
 252                                         $expected_ret = 0;
 253                                 } else {
 254                                         print "n:$name/$2\n";
 255                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 256                                         $msg_ops->end_test($msg_state, $2, $1, 1);
 257                                 }
 258                         } elsif ($1 eq "skip") {
 259                                 delete $open_tests->{$2};
 260                                 $msg_ops->end_test($msg_state, $2, $1, 0);
 261                         } elsif ($1 eq "error") {
 262                                 $statistics->{TESTS_ERROR}++;
 263                                 delete $open_tests->{$2};
 264                                 $msg_ops->end_test($msg_state, $2, $1, 1);
 265                         }
 266                 }
 267         }
 268
 269         foreach (keys %$open_tests) {
 270                 $msg_ops->end_test($msg_state, $_, "error", 1);
 271                 $msg_ops->output_msg($msg_state, "$_ was started but never finished!");
 272                 $statistics->{TESTS_ERROR}++;
 273         }
 274         my $ret = close(RESULT);
 275
 276         cleanup_pcap($msg_state,  $expected_ret, $ret);
 277
 278         $msg_ops->end_testsuite($msg_state, $expected_ret, $ret,
 279                                                         getlog_env($msg_state->{ENVNAME}));
 280
 281         if (not $opt_socket_wrapper_keep_pcap and
 282                 defined($msg_state->{PCAP_FILE})) {
 283                 $msg_ops->output_msg($msg_state,
 284                         "PCAP FILE: $msg_state->{PCAP_FILE}\n");
 285         }
 286
 287         if ($ret != $expected_ret) {
 288                 $statistics->{SUITES_FAIL}++;
 289                 exit(1) if ($opt_one);
 290         } else {
 291                 $statistics->{SUITES_OK}++;
 292         }
 293
 294         return ($ret == $expected_ret);
 295 }
 296
 297 sub ShowHelp()
 298 {
 299         print "Samba test runner
 300 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 301
 302 Usage: $Script [OPTIONS] PREFIX
 303
 304 Generic options:
 305  --help                     this help page
 306  --target=samba4|samba3|win Samba version to target
 307
 308 Paths:
 309  --prefix=DIR               prefix to run tests in [st]
 310  --srcdir=DIR               source directory [.]
 311  --builddir=DIR             output directory [.]
 312
 313 Target Specific:
 314  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 315  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 316                             failed
 317  --socket-wrapper           enable socket wrapper
 318  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 319
 320 Samba4 Specific:
 321  --ldap=openldap|fedora-ds     back smbd onto specified ldap server
 322
 323 Samba3 Specific:
 324  --bindir=PATH              path to binaries
 325
 326 Behaviour:
 327  --quick                    run quick overall test
 328  --one                      abort when the first test fails
 329  --immediate                print test output for failed tests during run
 330  --verbose                  be verbose
 331  --analyse-cmd CMD          command to run after each test
 332 ";
 333         exit(0);
 334 }
 335
 336 my $result = GetOptions (
 337                 'help|h|?' => \$opt_help,
 338                 'target=s' => \$opt_target,
 339                 'prefix=s' => \$prefix,
 340                 'socket-wrapper' => \$opt_socket_wrapper,
 341                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 342                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 343                 'quick' => \$opt_quick,
 344                 'one' => \$opt_one,
 345                 'immediate' => \$opt_immediate,
 346                 'expected-failures=s' => \$opt_expected_failures,
 347                 'skip=s' => \$opt_skip,
 348                 'srcdir=s' => \$srcdir,
 349                 'builddir=s' => \$builddir,
 350                 'verbose' => \$opt_verbose,
 351                 'testenv' => \$opt_testenv,
 352                 'ldap:s' => \$ldap,
 353                 'analyse-cmd=s' => \$opt_analyse_cmd,
 354                 'no-lazy-setup' => \$opt_no_lazy_setup,
 355                 'resetup-environment' => \$opt_resetup_env,
 356                 'bindir:s' => \$opt_bindir,
 357             );
 358
 359 exit(1) if (not $result);
 360
 361 ShowHelp() if ($opt_help);
 362
 363 my $tests = shift;
 364
 365 # quick hack to disable rpc validation when using valgrind - its way too slow
 366 unless (defined($ENV{VALGRIND})) {
 367         $ENV{VALIDATE} = "validate";
 368         $ENV{MALLOC_CHECK_} = 2;
 369 }
 370
 371 my $old_pwd = "$RealBin/..";
 372
 373 # Backwards compatibility:
 374 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 375         if (defined($ENV{FEDORA_DS_PREFIX})) {
 376                 $ldap = "fedora-ds";
 377         } else {
 378                 $ldap = "openldap";
 379         }
 380 }
 381
 382 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 383 if ($ldap) {
 384         # LDAP is slow
 385         $torture_maxtime *= 2;
 386 }
 387
 388 $prefix =~ s+//+/+;
 389 $prefix =~ s+/./+/+;
 390 $prefix =~ s+/$++;
 391
 392 die("using an empty prefix isn't allowed") unless $prefix ne "";
 393
 394 #Ensure we have the test prefix around
 395 mkdir($prefix, 0777) unless -d $prefix;
 396
 397 my $prefix_abs = abs_path($prefix);
 398 my $srcdir_abs = abs_path($srcdir);
 399
 400 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 401 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 402
 403 $ENV{PREFIX} = $prefix;
 404 $ENV{PREFIX_ABS} = $prefix_abs;
 405 $ENV{SRCDIR} = $srcdir;
 406 $ENV{SRCDIR_ABS} = $srcdir_abs;
 407
 408 my $tls_enabled = not $opt_quick;
 409 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 410                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 411
 412 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 413 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 414 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 415 if (defined($ENV{LD_LIBRARY_PATH})) {
 416         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 417 } else {
 418         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 419 }
 420 if (defined($ENV{PKG_CONFIG_PATH})) {
 421         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 422 } else {
 423         $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig";
 424 }
 425 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 426
 427
 428 if ($opt_socket_wrapper_pcap) {
 429         # Socket wrapper pcap implies socket wrapper
 430         $opt_socket_wrapper = 1;
 431 }
 432
 433 my $socket_wrapper_dir;
 434 if ($opt_socket_wrapper) {
 435         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 436         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 437 } else {
 438         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 439 }
 440
 441 my $target;
 442
 443 if ($opt_target eq "samba4") {
 444         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 445 } elsif ($opt_target eq "samba3") {
 446         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 447                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 448         }
 449
 450         $target = new Samba3($opt_bindir);
 451 } elsif ($opt_target eq "win") {
 452         die("Windows tests will not run with socket wrapper enabled.")
 453                 if ($opt_socket_wrapper);
 454         $target = new Windows();
 455 }
 456
 457 if (defined($opt_expected_failures)) {
 458         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 459         while (<KNOWN>) {
 460                 chomp;
 461                 s/([ \t]+)\#(.*)$//;
 462                 push (@expected_failures, $_); }
 463         close(KNOWN);
 464 }
 465
 466 if (defined($opt_skip)) {
 467         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 468         while (<SKIP>) {
 469                 chomp;
 470                 s/([ \t]+)\#(.*)$//;
 471                 push (@skips, $_); }
 472         close(SKIP);
 473 }
 474
 475 my $interfaces = join(',', ("127.0.0.6/8",
 476                             "127.0.0.7/8",
 477                             "127.0.0.8/8",
 478                             "127.0.0.9/8",
 479                             "127.0.0.10/8",
 480                             "127.0.0.11/8"));
 481
 482 my $conffile = "$prefix_abs/client/client.conf";
 483
 484 sub write_clientconf($$)
 485 {
 486         my ($conffile, $vars) = @_;
 487
 488         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 489
 490         if ( -d "$prefix/client/private" ) {
 491                 unlink <$prefix/client/private/*>;
 492         } else {
 493                 mkdir("$prefix/client/private", 0777);
 494         }
 495
 496         open(CF, ">$conffile");
 497         print CF "[global]\n";
 498         if (defined($ENV{VALGRIND})) {
 499                 print CF "\ticonv:native = true\n";
 500         } else {
 501                 print CF "\ticonv:native = false\n";
 502         }
 503         print CF "\tnetbios name = client\n";
 504         if (defined($vars->{DOMAIN})) {
 505                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 506         }
 507         if (defined($vars->{REALM})) {
 508                 print CF "\trealm = $vars->{REALM}\n";
 509         }
 510         if (defined($vars->{NCALRPCDIR})) {
 511                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 512         }
 513         if (defined($vars->{PIDDIR})) {
 514                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 515         }
 516         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 517                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 518         }
 519         print CF "
 520         private dir = $prefix_abs/client/private
 521         js include = $srcdir_abs/scripting/libjs
 522         name resolve order = bcast
 523         interfaces = $interfaces
 524         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 525         max xmit = 32K
 526         notify:inotify = false
 527         ldb:nosync = true
 528         system:anonymous = true
 529         torture:basedir = $prefix_abs/client
 530 #We don't want to pass our self-tests if the PAC code is wrong
 531         gensec:require_pac = true
 532 ";
 533         close(CF);
 534 }
 535
 536
 537 my @torture_options = ();
 538 push (@torture_options, "--configfile=$conffile");
 539 # ensure any one smbtorture call doesn't run too long
 540 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 541 push (@torture_options, "--target=$opt_target");
 542 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 543 push (@torture_options, "--format=subunit");
 544 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 545
 546 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 547 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 548
 549 my @todo = ();
 550
 551 my $testsdir = "$srcdir/selftest";
 552 $ENV{SMB_CONF_PATH} = "$conffile";
 553 $ENV{CONFIGURATION} = "--configfile=$conffile";
 554
 555 my %required_envs = ();
 556
 557 if ($opt_quick) {
 558         open(IN, "$testsdir/tests_quick.sh|");
 559 } else {
 560         open(IN, "$testsdir/tests_all.sh|");
 561 }
 562 while (<IN>) {
 563         if ($_ eq "-- TEST --\n") {
 564                 my $name = <IN>;
 565                 $name =~ s/\n//g;
 566                 my $env = <IN>;
 567                 $env =~ s/\n//g;
 568                 my $cmdline = <IN>;
 569                 $cmdline =~ s/\n//g;
 570                 if (not defined($tests) or $name =~ /$tests/) {
 571                         $required_envs{$env} = 1;
 572                         push (@todo, [$name, $env, $cmdline]);
 573                 }
 574         } else {
 575                 print;
 576         }
 577 }
 578 close(IN) or die("Error creating recipe");
 579
 580 my $suitestotal = $#todo + 1;
 581 my $i = 0;
 582 $| = 1;
 583
 584 my %running_envs = ();
 585
 586 my @exported_envvars = (
 587         # domain stuff
 588         "DOMAIN",
 589         "REALM",
 590
 591         # domain controller stuff
 592         "DC_SERVER",
 593         "DC_SERVER_IP",
 594         "DC_NETBIOSNAME",
 595         "DC_NETBIOSALIAS",
 596
 597         # server stuff
 598         "SERVER",
 599         "SERVER_IP",
 600         "NETBIOSNAME",
 601         "NETBIOSALIAS",
 602
 603         # user stuff
 604         "USERNAME",
 605         "PASSWORD",
 606         "DC_USERNAME",
 607         "DC_PASSWORD",
 608
 609         # misc stuff
 610         "KRB5_CONFIG",
 611         "WINBINDD_SOCKET_DIR"
 612 );
 613
 614 sub setup_env($)
 615 {
 616         my ($envname) = @_;
 617
 618         my $testenv_vars;
 619         if ($envname eq "none") {
 620                 $testenv_vars = {};
 621         } elsif (defined($running_envs{$envname})) {
 622                 $testenv_vars = $running_envs{$envname};
 623                 if (not $target->check_env($testenv_vars)) {
 624                         $testenv_vars = undef;
 625                 }
 626         } else {
 627                 $testenv_vars = $target->setup_env($envname, $prefix);
 628         }
 629
 630         return undef unless defined($testenv_vars);
 631
 632         SocketWrapper::set_default_iface(6);
 633         write_clientconf($conffile, $testenv_vars);
 634
 635         foreach (@exported_envvars) {
 636                 if (defined($testenv_vars->{$_})) {
 637                         $ENV{$_} = $testenv_vars->{$_};
 638                 } else {
 639                         delete $ENV{$_};
 640                 }
 641         }
 642
 643         $running_envs{$envname} = $testenv_vars;
 644         return $testenv_vars;
 645 }
 646
 647 sub exported_envvars_str($)
 648 {
 649         my ($testenv_vars) = @_;
 650         my $out = "";
 651
 652         foreach (@exported_envvars) {
 653                 next unless defined($testenv_vars->{$_});
 654                 $out .= $_."=".$testenv_vars->{$_}."\n";
 655         }
 656
 657         return $out;
 658 }
 659
 660 sub getlog_env($)
 661 {
 662         my ($envname) = @_;
 663         return "" if ($envname eq "none");
 664         return $target->getlog_env($running_envs{$envname});
 665 }
 666
 667 sub check_env($)
 668 {
 669         my ($envname) = @_;
 670         return 1 if ($envname eq "none");
 671         return $target->check_env($running_envs{$envname});
 672 }
 673
 674 sub teardown_env($)
 675 {
 676         my ($envname) = @_;
 677         return if ($envname eq "none");
 678         $target->teardown_env($running_envs{$envname});
 679         delete $running_envs{$envname};
 680 }
 681
 682 my $msg_ops;
 683 if ($from_build_farm) {
 684         require output::buildfarm;
 685         $msg_ops = new output::buildfarm();
 686 } else {
 687         require output::plain;
 688         $msg_ops = new output::plain($opt_verbose, $opt_immediate, $statistics);
 689 }
 690
 691 if ($opt_no_lazy_setup) {
 692         setup_env($_) foreach (keys %required_envs);
 693 }
 694
 695 if ($opt_testenv) {
 696         my $testenv_name = $ENV{SELFTEST_TESTENV};
 697         $testenv_name = "dc" unless defined($testenv_name);
 698
 699         my $testenv_vars = setup_env($testenv_name);
 700
 701         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 702
 703         my $envvarstr = exported_envvars_str($testenv_vars);
 704
 705         my $term = ($ENV{TERM} or "xterm");
 706         system("$term -e 'echo -e \"
 707 Welcome to the Samba4 Test environment '$testenv_name'
 708
 709 This matches the client environment used in make test
 710 smbd is pid `cat \$PIDDIR/smbd.pid`
 711
 712 Some useful environment variables:
 713 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 714 CONFIGURATION=\$CONFIGURATION
 715
 716 $envvarstr
 717 \" && bash'");
 718         teardown_env($testenv_name);
 719 } else {
 720         foreach (@todo) {
 721                 $i++;
 722                 my $cmd = $$_[2];
 723                 $cmd =~ s/([\(\)])/\\$1/g;
 724                 my $name = $$_[0];
 725                 my $envname = $$_[1];
 726
 727                 if (skip($name)) {
 728                         print "SKIPPED: $name\n";
 729                         $statistics->{SUITES_SKIPPED}++;
 730                         next;
 731                 }
 732
 733                 my $envvars = setup_env($envname);
 734                 if (not defined($envvars)) {
 735                         $statistics->{SUITES_FAIL}++;
 736                         $statistics->{TESTS_ERROR}++;
 737                         $msg_ops->missing_env($name, $envname);
 738                         next;
 739                 }
 740
 741                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 742
 743                 if (defined($opt_analyse_cmd)) {
 744                         system("$opt_analyse_cmd \"$name\"");
 745                 }
 746
 747                 teardown_env($envname) if ($opt_resetup_env);
 748         }
 749 }
 750
 751 print "\n";
 752
 753 teardown_env($_) foreach (keys %running_envs);
 754
 755 $target->stop();
 756
 757 $statistics->{END_TIME} = time();
 758 my $duration = ($statistics->{END_TIME}-$statistics->{START_TIME});
 759 my $numfailed = $statistics->{SUITES_FAIL};
 760 if ($numfailed == 0) {
 761         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 762                  $statistics->{TESTS_EXPECTED_FAIL};
 763         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 764 } else {
 765         $msg_ops->summary();
 766 }
 767 print "DURATION: $duration seconds\n";
 768
 769 my $failed = 0;
 770
 771 # if there were any valgrind failures, show them
 772 foreach (<$prefix/valgrind.log*>) {
 773         next unless (-s $_);
 774         system("grep DWARF2.CFI.reader $_ > /dev/null");
 775         if ($? >> 8 == 0) {
 776             print "VALGRIND FAILURE\n";
 777             $failed++;
 778             system("cat $_");
 779         }
 780 }
 781
 782 if ($from_build_farm) {
 783         print "TEST STATUS: $numfailed\n";
 784 }
 785
 786 exit $numfailed;