source/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--one>
  73
  74 Abort as soon as one test fails.
  75
  76 =back
  77
  78 =head1 ENVIRONMENT
  79
  80 =over 4
  81
  82 =item I<SMBD_VALGRIND>
  83
  84 =item I<TORTURE_MAXTIME>
  85
  86 =item I<VALGRIND>
  87
  88 =item I<TEST_LDAP>
  89
  90 =item I<TLS_ENABLED>
  91
  92 =item I<srcdir>
  93
  94 =back
  95
  96 =head1 LICENSE
  97
  98 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
  99
 100 =head1 AUTHOR
 101
 102 Jelmer Vernooij
 103
 104 =cut
 105
 106 use strict;
 107 use warnings;
 108
 109 use FindBin qw($RealBin $Script);
 110 use File::Spec;
 111 use Getopt::Long;
 112 use POSIX;
 113 use Cwd;
 114 use lib "$RealBin";
 115 use Samba4;
 116 use SocketWrapper;
 117
 118 my $opt_help = 0;
 119 my $opt_target = "samba4";
 120 my $opt_quick = 0;
 121 my $opt_socket_wrapper = 0;
 122 my $opt_socket_wrapper_pcap = undef;
 123 my $opt_one = 0;
 124 my $opt_immediate = 0;
 125 my $opt_expected_failures = undef;
 126 my $opt_verbose = 0;
 127
 128 my $srcdir = ".";
 129 my $builddir = ".";
 130 my $prefix = "st";
 131
 132 my $suitesfailed = [];
 133 my $start = time();
 134 my @expected_failures = ();
 135
 136 my $statistics = {
 137         SUITES_FAIL => 0,
 138         SUITES_OK => 0,
 139
 140         TESTS_UNEXPECTED_OK => 0,
 141         TESTS_EXPECTED_OK => 0,
 142         TESTS_UNEXPECTED_FAIL => 0,
 143         TESTS_EXPECTED_FAIL => 0,
 144         TESTS_ERROR => 0
 145 };
 146
 147 sub expecting_failure($)
 148 {
 149         my $fullname = shift;
 150
 151         foreach (@expected_failures) {
 152                 return 1 if $fullname =~ /^$_$/;
 153         }
 154
 155         return 0;
 156 }
 157
 158 sub run_test_buildfarm($$$$)
 159 {
 160         my ($name, $cmd, $i, $suitestotal) = @_;
 161         print "--==--==--==--==--==--==--==--==--==--==--\n";
 162         print "Running test $name (level 0 stdout)\n";
 163         print "--==--==--==--==--==--==--==--==--==--==--\n";
 164         system("date");
 165
 166         my $expected_ret = 1;
 167         my $open_tests = {};
 168         open(RESULT, "$cmd|");
 169         while (<RESULT>) {
 170                 print;
 171                 if (/^test: (.+)\n/) {
 172                         $open_tests->{$1} = 1;
 173                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 174                         my $result = $1;
 175                         if ($1 eq "success") {
 176                                 delete $open_tests->{$2};
 177                                 if (expecting_failure("$name/$2")) {
 178                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 179                                 } else {
 180                                         $statistics->{TESTS_EXPECTED_OK}++;
 181                                 }
 182                         } elsif ($1 eq "failure") {
 183                                 delete $open_tests->{$2};
 184                                 if (expecting_failure("$name/$2")) {
 185                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 186                                         $expected_ret = 0;
 187                                 } else {
 188                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 189                                 }
 190                         } elsif ($1 eq "skip") {
 191                                 delete $open_tests->{$2};
 192                         } elsif ($1 eq "error") {
 193                                 $statistics->{TESTS_ERROR}++;
 194                                 delete $open_tests->{$2};
 195                         }
 196                 }
 197         }
 198         print "COMMAND: $cmd\n";
 199         foreach (keys %$open_tests) {
 200                 print "$_ was started but never finished!\n";
 201                 $statistics->{TESTS_ERROR}++;
 202         }
 203         my $ret = close(RESULT);
 204
 205         print "==========================================\n";
 206         if ($ret == $expected_ret) {
 207                 print "TEST PASSED: $name\n";
 208         } else {
 209                 print "TEST FAILED: $name (status $ret)\n";
 210         }
 211         print "==========================================\n";
 212 }
 213
 214 my $test_output = {};
 215 sub run_test_plain($$$$)
 216 {
 217         my ($name, $cmd, $i, $totalsuites) = @_;
 218         my $err = "";
 219         if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
 220         printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
 221         open(RESULT, "$cmd 2>&1|");
 222         my $expected_ret = 1;
 223         my $open_tests = {};
 224         $test_output->{$name} = "";
 225         while (<RESULT>) {
 226                 $test_output->{$name}.=$_;
 227                 print if ($opt_verbose);
 228                 if (/^test: (.+)\n/) {
 229                         $open_tests->{$1} = 1;
 230                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 231                         my $result = $1;
 232                         if ($1 eq "success") {
 233                                 delete $open_tests->{$2};
 234                                 if (expecting_failure("$name/$2")) {
 235                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 236                                 } else {
 237                                         $statistics->{TESTS_EXPECTED_OK}++;
 238                                 }
 239                         } elsif ($1 eq "failure") {
 240                                 delete $open_tests->{$2};
 241                                 if (expecting_failure("$name/$2")) {
 242                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 243                                         $expected_ret = 0;
 244                                 } else {
 245                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 246                                 }
 247                         } elsif ($1 eq "skip") {
 248                                 delete $open_tests->{$2};
 249                         } elsif ($1 eq "error") {
 250                                 $statistics->{TESTS_ERROR}++;
 251                                 delete $open_tests->{$2};
 252                         }
 253                 }
 254         }
 255         $test_output->{$name}.="COMMAND: $cmd\n";
 256         foreach (keys %$open_tests) {
 257                 $test_output->{$name}.="$_ was started but never finished!\n";
 258                 $statistics->{TESTS_ERROR}++;
 259         }
 260         my $ret = close(RESULT);
 261         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 262                 print "$test_output->{$name}\n";
 263         }
 264         if ($ret != $expected_ret) {
 265                 push(@$suitesfailed, $name);
 266                 $statistics->{SUITES_FAIL}++;
 267                 exit(1) if ($opt_one);
 268         } else {
 269                 $statistics->{SUITES_OK}++;
 270         }
 271 }
 272
 273 sub ShowHelp()
 274 {
 275         print "Samba test runner
 276 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 277
 278 Usage: $Script [OPTIONS] PREFIX
 279
 280 Generic options:
 281  --help                     this help page
 282
 283 Paths:
 284  --prefix=DIR               prefix to run tests in [st]
 285  --srcdir=DIR               source directory [.]
 286  --builddir=DIR             output directory [.]
 287
 288 Target Specific:
 289  --target=samba4|samba3|win Samba version to target
 290  --socket-wrapper-pcap=FILE save traffic to pcap file
 291  --socket-wrapper           enable socket wrapper
 292  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 293
 294 Behaviour:
 295  --quick                    run quick overall test
 296  --one                      abort when the first test fails
 297  --immediate                print test output for failed tests during run
 298  --verbose                  be verbose
 299 ";
 300         exit(0);
 301 }
 302
 303 my $result = GetOptions (
 304             'help|h|?' => \$opt_help,
 305                 'target=s' => \$opt_target,
 306                 'prefix=s' => \$prefix,
 307                 'socket-wrapper' => \$opt_socket_wrapper,
 308                 'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
 309                 'quick' => \$opt_quick,
 310                 'one' => \$opt_one,
 311                 'immediate' => \$opt_immediate,
 312                 'expected-failures=s' => \$opt_expected_failures,
 313                 'srcdir=s' => \$srcdir,
 314                 'builddir=s' => \$builddir,
 315                 'verbose' => \$opt_verbose
 316             );
 317
 318 exit(1) if (not $result);
 319
 320 ShowHelp() if ($opt_help);
 321
 322 my $tests = shift;
 323
 324 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
 325 unless (defined($torture_maxtime)) {
 326         $torture_maxtime = 1200;
 327 }
 328
 329 # quick hack to disable rpc validation when using valgrind - its way too slow
 330 unless (defined($ENV{VALGRIND})) {
 331         $ENV{VALIDATE} = "validate";
 332 }
 333
 334 my $old_pwd = "$RealBin/../..";
 335 my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
 336
 337 $prefix =~ s+//+/+;
 338 $ENV{PREFIX} = $prefix;
 339
 340 $ENV{SRCDIR} = $srcdir;
 341
 342 my $bindir = "$srcdir/bin";
 343 my $setupdir = "$srcdir/setup";
 344 my $testsdir = "$srcdir/script/tests";
 345
 346 my $tls_enabled = not $opt_quick;
 347 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 348                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 349
 350 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 351 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 352 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 353 if (defined($ENV{LD_LIBRARY_PATH})) {
 354         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 355 } else {
 356         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 357 }
 358 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 359 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 360
 361 my @torture_options = ();
 362
 363 my $testenv_vars = {};
 364
 365 if ($opt_target eq "samba4") {
 366         $testenv_vars = Samba4::provision($prefix);
 367 } elsif ($opt_target eq "win") {
 368         die ("Windows tests will not run without root privileges.")
 369                 if (`whoami` ne "root");
 370
 371         die("Windows tests will not run with socket wrapper enabled.")
 372         if ($opt_socket_wrapper);
 373
 374         die("Windows tests will not run quickly.") if ($opt_quick);
 375
 376         die("Environment variable WINTESTCONF has not been defined.\n".
 377                 "Windows tests will not run unconfigured.") if (not defined($ENV{WINTESTCONF}));
 378
 379         die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
 380
 381         $ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
 382 } elsif ($opt_target eq "none") {
 383 } else {
 384         die("unknown target `$opt_target'");
 385 }
 386
 387 foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
 388
 389 if ($opt_socket_wrapper_pcap) {
 390         $ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
 391         # Socket wrapper pcap implies socket wrapper
 392         $opt_socket_wrapper = 1;
 393 }
 394
 395 my $socket_wrapper_dir;
 396 if ($opt_socket_wrapper)
 397 {
 398         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
 399         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 400 }
 401
 402 # Start slapd before smbd
 403 if ($ldap) {
 404         Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
 405
 406     print "LDAP PROVISIONING...";
 407         Samba4::provision_ldap($bindir, $setupdir);
 408
 409     # LDAP is slow
 410         $torture_maxtime *= 2;
 411 }
 412
 413 if (defined($opt_expected_failures)) {
 414         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 415         while (<KNOWN>) {
 416                 chomp;
 417                 s/([ \t]+)\#(.*)$//;
 418                 push (@expected_failures, $_); }
 419         close(KNOWN);
 420 }
 421
 422 my $test_fifo = "$prefix/smbd_test.fifo";
 423
 424 $ENV{SMBD_TEST_FIFO} = $test_fifo;
 425 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
 426
 427 SocketWrapper::set_default_iface(1);
 428 my $max_time = 5400;
 429 if (defined($ENV{SMBD_MAX_TIME})) {
 430         $max_time = $ENV{SMBD_MAX_TIME};
 431 }
 432 Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG},
 433                         $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
 434
 435 SocketWrapper::set_default_iface(6);
 436
 437 my $interfaces = join(',', ("127.0.0.6/8",
 438                                  "127.0.0.7/8",
 439                                                  "127.0.0.8/8",
 440                                                  "127.0.0.9/8",
 441                                                  "127.0.0.10/8",
 442                                                  "127.0.0.11/8"));
 443
 444 push (@torture_options, "--option=interfaces=$interfaces");
 445 push (@torture_options, $ENV{CONFIGURATION});
 446 # ensure any one smbtorture call doesn't run too long
 447 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 448 push (@torture_options, "--target=$opt_target");
 449 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 450 push (@torture_options, "--format=subunit");
 451 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 452
 453 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 454 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 455
 456 open(DATA, ">$test_fifo");
 457
 458 my @todo = ();
 459
 460 if ($opt_target eq "win") {
 461         system("$testsdir/test_win.sh");
 462 } else {
 463         if ($opt_quick) {
 464                 open(IN, "$testsdir/tests_quick.sh|");
 465         } else {
 466                 open(IN, "$testsdir/tests_all.sh|");
 467         }
 468         while (<IN>) {
 469                 if ($_ eq "-- TEST --\n") {
 470                         my $name = <IN>;
 471                         $name =~ s/\n//g;
 472                         my $cmdline = <IN>;
 473                         $cmdline =~ s/\n//g;
 474                         push (@todo, [$name, $cmdline])
 475                                 if (not defined($tests) or $name =~ /$tests/);
 476                 } else {
 477                         print;
 478                 }
 479         }
 480         close(IN) or die("Error creating recipe");
 481 }
 482
 483 Samba4::wait_for_start();
 484
 485 # start off with 0 failures
 486 $ENV{failed} = 0;
 487
 488 my $suitestotal = $#todo + 1;
 489 my $i = 0;
 490 $| = 1;
 491
 492 delete $ENV{DOMAIN};
 493
 494 foreach (@todo) {
 495         $i++;
 496         my $cmd = $$_[1];
 497         $cmd =~ s/([\(\)])/\\$1/g;
 498         my $name = $$_[0];
 499         if ($from_build_farm) {
 500                 run_test_buildfarm($name, $cmd, $i, $suitestotal);
 501         } else {
 502                 run_test_plain($name, $cmd, $i, $suitestotal);
 503         }
 504 }
 505
 506 print "\n";
 507
 508 close(DATA);
 509
 510 sleep(2);
 511
 512 my $failed = $? >> 8;
 513
 514 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
 515         open(IN, "<$ENV{PIDDIR}/smbd.pid") or die("unable to open smbd pid file");
 516         kill 9, <IN>;
 517         close(IN);
 518 }
 519
 520 Samba4::slapd_stop() if ($ldap);
 521
 522 my $end = time();
 523 my $duration = ($end-$start);
 524 my $numfailed = $#$suitesfailed+1;
 525 if ($numfailed == 0) {
 526         my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
 527         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 528 } else {
 529
 530         unless ($from_build_farm) {
 531                 if (not $opt_immediate and not $opt_verbose) {
 532                         foreach (@$suitesfailed) {
 533                                 print "===============================================================================\n";
 534                                 print "FAIL: $_\n";
 535                                 print $test_output->{$_};
 536                                 print "\n";
 537                         }
 538                 }
 539
 540                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 541         } else {
 542                 print <<EOF
 543 ************************
 544 *** TESTSUITE FAILED ***
 545 ************************
 546 EOF
 547 ;
 548         }
 549 }
 550 print "DURATION: $duration seconds\n";
 551
 552 # if there were any valgrind failures, show them
 553 foreach (<$prefix/valgrind.log*>) {
 554         next unless (-s $_);
 555         system("grep DWARF2.CFI.reader $_ > /dev/null");
 556         if ($? >> 8 == 0) {
 557             print "VALGRIND FAILURE\n";
 558             $failed++;
 559             system("cat $_");
 560         }
 561 }
 562
 563 exit $numfailed;