source4/torture/smbtorture.c

   1 /*
   2    Unix SMB/CIFS implementation.
   3    SMB torture tester
   4    Copyright (C) Andrew Tridgell 1997-2003
   5    Copyright (C) Jelmer Vernooij 2006
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 2 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20 */
  21
  22 #include "includes.h"
  23 #include "lib/cmdline/popt_common.h"
  24 #include "system/time.h"
  25 #include "system/wait.h"
  26 #include "system/filesys.h"
  27 #include "system/readline.h"
  28 #include "lib/smbreadline/smbreadline.h"
  29 #include "libcli/libcli.h"
  30 #include "lib/ldb/include/ldb.h"
  31 #include "lib/events/events.h"
  32 #include "dynconfig.h"
  33
  34 #include "torture/torture.h"
  35 #include "build.h"
  36 #include "lib/util/dlinklist.h"
  37 #include "librpc/rpc/dcerpc.h"
  38
  39 static bool run_matching(struct torture_context *torture,
  40                                                  const char *prefix,
  41                                                  const char *expr,
  42                                                  struct torture_suite *suite,
  43                                                  bool *matched)
  44 {
  45         bool ret = true;
  46
  47         if (suite == NULL) {
  48                 struct torture_suite *o;
  49
  50                 for (o = torture_root->children; o; o = o->next) {
  51                         if (gen_fnmatch(expr, o->name) == 0) {
  52                                 *matched = true;
  53                                 init_iconv();
  54                                 ret &= torture_run_suite(torture, o);
  55                                 continue;
  56                         }
  57
  58                         ret &= run_matching(torture, o->name, expr, o, matched);
  59                 }
  60         } else {
  61                 char *name;
  62                 struct torture_suite *c;
  63                 struct torture_tcase *t;
  64
  65                 for (c = suite->children; c; c = c->next) {
  66                         asprintf(&name, "%s-%s", prefix, c->name);
  67
  68                         if (gen_fnmatch(expr, name) == 0) {
  69                                 *matched = true;
  70                                 init_iconv();
  71                                 torture->active_testname = talloc_strdup(torture, prefix);
  72                                 ret &= torture_run_suite(torture, c);
  73                                 free(name);
  74                                 continue;
  75                         }
  76
  77                         ret &= run_matching(torture, name, expr, c, matched);
  78
  79                         free(name);
  80                 }
  81
  82                 for (t = suite->testcases; t; t = t->next) {
  83                         asprintf(&name, "%s-%s", prefix, t->name);
  84                         if (gen_fnmatch(expr, name) == 0) {
  85                                 *matched = true;
  86                                 init_iconv();
  87                                 torture->active_testname = talloc_strdup(torture, prefix);
  88                                 ret &= torture_run_tcase(torture, t);
  89                                 talloc_free(torture->active_testname);
  90                         }
  91                         free(name);
  92                 }
  93         }
  94
  95         return ret;
  96 }
  97
  98 #define MAX_COLS 80 /* FIXME: Determine this at run-time */
  99
 100 /****************************************************************************
 101 run a specified test or "ALL"
 102 ****************************************************************************/
 103 static bool run_test(struct torture_context *torture, const char *name)
 104 {
 105         bool ret = true;
 106         bool matched = false;
 107         struct torture_suite *o;
 108
 109         if (strequal(name, "ALL")) {
 110                 for (o = torture_root->children; o; o = o->next) {
 111                         ret &= torture_run_suite(torture, o);
 112                 }
 113                 return ret;
 114         }
 115
 116         ret = run_matching(torture, NULL, name, NULL, &matched);
 117
 118         if (!matched) {
 119                 printf("Unknown torture operation '%s'\n", name);
 120                 return false;
 121         }
 122
 123         return ret;
 124 }
 125
 126 static void parse_dns(const char *dns)
 127 {
 128         char *userdn, *basedn, *secret;
 129         char *p, *d;
 130
 131         /* retrievieng the userdn */
 132         p = strchr_m(dns, '#');
 133         if (!p) {
 134                 lp_set_cmdline("torture:ldap_userdn", "");
 135                 lp_set_cmdline("torture:ldap_basedn", "");
 136                 lp_set_cmdline("torture:ldap_secret", "");
 137                 return;
 138         }
 139         userdn = strndup(dns, p - dns);
 140         lp_set_cmdline("torture:ldap_userdn", userdn);
 141
 142         /* retrieve the basedn */
 143         d = p + 1;
 144         p = strchr_m(d, '#');
 145         if (!p) {
 146                 lp_set_cmdline("torture:ldap_basedn", "");
 147                 lp_set_cmdline("torture:ldap_secret", "");
 148                 return;
 149         }
 150         basedn = strndup(d, p - d);
 151         lp_set_cmdline("torture:ldap_basedn", basedn);
 152
 153         /* retrieve the secret */
 154         p = p + 1;
 155         if (!p) {
 156                 lp_set_cmdline("torture:ldap_secret", "");
 157                 return;
 158         }
 159         secret = strdup(p);
 160         lp_set_cmdline("torture:ldap_secret", secret);
 161
 162         printf ("%s - %s - %s\n", userdn, basedn, secret);
 163
 164 }
 165
 166 static void usage(poptContext pc)
 167 {
 168         struct torture_suite *o;
 169         struct torture_suite *s;
 170         struct torture_tcase *t;
 171         int i;
 172
 173         poptPrintUsage(pc, stdout, 0);
 174         printf("\n");
 175
 176         printf("The binding format is:\n\n");
 177
 178         printf("  TRANSPORT:host[flags]\n\n");
 179
 180         printf("  where TRANSPORT is either ncacn_np for SMB, ncacn_ip_tcp for RPC/TCP\n");
 181         printf("  or ncalrpc for local connections.\n\n");
 182
 183         printf("  'host' is an IP or hostname or netbios name. If the binding string\n");
 184         printf("  identifies the server side of an endpoint, 'host' may be an empty\n");
 185         printf("  string.\n\n");
 186
 187         printf("  'flags' can include a SMB pipe name if using the ncacn_np transport or\n");
 188         printf("  a TCP port number if using the ncacn_ip_tcp transport, otherwise they\n");
 189         printf("  will be auto-determined.\n\n");
 190
 191         printf("  other recognised flags are:\n\n");
 192
 193         printf("    sign : enable ntlmssp signing\n");
 194         printf("    seal : enable ntlmssp sealing\n");
 195         printf("    connect : enable rpc connect level auth (auth, but no sign or seal)\n");
 196         printf("    validate: enable the NDR validator\n");
 197         printf("    print: enable debugging of the packets\n");
 198         printf("    bigendian: use bigendian RPC\n");
 199         printf("    padcheck: check reply data for non-zero pad bytes\n\n");
 200
 201         printf("  For example, these all connect to the samr pipe:\n\n");
 202
 203         printf("    ncacn_np:myserver\n");
 204         printf("    ncacn_np:myserver[samr]\n");
 205         printf("    ncacn_np:myserver[\\pipe\\samr]\n");
 206         printf("    ncacn_np:myserver[/pipe/samr]\n");
 207         printf("    ncacn_np:myserver[samr,sign,print]\n");
 208         printf("    ncacn_np:myserver[\\pipe\\samr,sign,seal,bigendian]\n");
 209         printf("    ncacn_np:myserver[/pipe/samr,seal,validate]\n");
 210         printf("    ncacn_np:\n");
 211         printf("    ncacn_np:[/pipe/samr]\n\n");
 212
 213         printf("    ncacn_ip_tcp:myserver\n");
 214         printf("    ncacn_ip_tcp:myserver[1024]\n");
 215         printf("    ncacn_ip_tcp:myserver[1024,sign,seal]\n\n");
 216
 217         printf("    ncalrpc:\n\n");
 218
 219         printf("The UNC format is:\n\n");
 220
 221         printf("  //server/share\n\n");
 222
 223         printf("Tests are:");
 224
 225         for (o = torture_root->children; o; o = o->next) {
 226                 printf("\n%s (%s):\n  ", o->description, o->name);
 227
 228                 i = 0;
 229                 for (s = o->children; s; s = s->next) {
 230                         if (i + strlen(o->name) + strlen(s->name) >= (MAX_COLS - 3)) {
 231                                 printf("\n  ");
 232                                 i = 0;
 233                         }
 234                         i+=printf("%s-%s ", o->name, s->name);
 235                 }
 236
 237                 for (t = o->testcases; t; t = t->next) {
 238                         if (i + strlen(o->name) + strlen(t->name) >= (MAX_COLS - 3)) {
 239                                 printf("\n  ");
 240                                 i = 0;
 241                         }
 242                         i+=printf("%s-%s ", o->name, t->name);
 243                 }
 244
 245                 if (i) printf("\n");
 246         }
 247
 248         printf("\nThe default test is ALL.\n");
 249
 250         exit(1);
 251 }
 252
 253 static bool is_binding_string(const char *binding_string)
 254 {
 255         TALLOC_CTX *mem_ctx = talloc_named_const(NULL, 0, "is_binding_string");
 256         struct dcerpc_binding *binding_struct;
 257         NTSTATUS status;
 258
 259         status = dcerpc_parse_binding(mem_ctx, binding_string, &binding_struct);
 260
 261         talloc_free(mem_ctx);
 262         return NT_STATUS_IS_OK(status);
 263 }
 264
 265 static void max_runtime_handler(int sig)
 266 {
 267         DEBUG(0,("maximum runtime exceeded for smbtorture - terminating\n"));
 268         exit(1);
 269 }
 270
 271 struct timeval last_suite_started;
 272
 273 static void simple_suite_start(struct torture_context *ctx,
 274                                                            struct torture_suite *suite)
 275 {
 276         last_suite_started = timeval_current();
 277         printf("Running %s\n", suite->name);
 278 }
 279
 280 static void simple_suite_finish(struct torture_context *ctx,
 281                                                            struct torture_suite *suite)
 282 {
 283
 284         printf("%s took %g secs\n\n", suite->name,
 285                    timeval_elapsed(&last_suite_started));
 286 }
 287
 288 static void simple_test_result (struct torture_context *context,
 289                                                                 enum torture_result res, const char *reason)
 290 {
 291         switch (res) {
 292         case TORTURE_OK:
 293                 if (reason)
 294                         printf("OK: %s\n", reason);
 295                 break;
 296         case TORTURE_FAIL:
 297                 printf("TEST %s FAILED! - %s\n", context->active_test->name, reason);
 298                 break;
 299         case TORTURE_ERROR:
 300                 printf("ERROR IN TEST %s! - %s\n", context->active_test->name, reason);
 301                 break;
 302         case TORTURE_SKIP:
 303                 printf("SKIP: %s - %s\n", context->active_test->name, reason);
 304                 break;
 305         }
 306 }
 307
 308 static void simple_comment (struct torture_context *test,
 309                                                         const char *comment)
 310 {
 311         printf("%s", comment);
 312 }
 313
 314 const static struct torture_ui_ops std_ui_ops = {
 315         .comment = simple_comment,
 316         .suite_start = simple_suite_start,
 317         .suite_finish = simple_suite_finish,
 318         .test_result = simple_test_result
 319 };
 320
 321
 322 static void subunit_test_start (struct torture_context *ctx,
 323                                                             struct torture_tcase *tcase,
 324                                                                 struct torture_test *test)
 325 {
 326         printf("test: %s\n", test->name);
 327 }
 328
 329 static void subunit_test_result (struct torture_context *context,
 330                                                                  enum torture_result res, const char *reason)
 331 {
 332         switch (res) {
 333         case TORTURE_OK:
 334                 printf("success: %s", context->active_test->name);
 335                 break;
 336         case TORTURE_FAIL:
 337                 printf("failure: %s", context->active_test->name);
 338                 break;
 339         case TORTURE_ERROR:
 340                 printf("error: %s", context->active_test->name);
 341                 break;
 342         case TORTURE_SKIP:
 343                 printf("skip: %s", context->active_test->name);
 344                 break;
 345         }
 346         if (reason)
 347                 printf(" [ %s ]", reason);
 348         printf("\n");
 349 }
 350
 351 static void subunit_comment (struct torture_context *test,
 352                                                          const char *comment)
 353 {
 354         fprintf(stderr, "%s", comment);
 355 }
 356
 357 const static struct torture_ui_ops subunit_ui_ops = {
 358         .comment = subunit_comment,
 359         .test_start = subunit_test_start,
 360         .test_result = subunit_test_result
 361 };
 362
 363 static void harness_test_start (struct torture_context *ctx,
 364                                                             struct torture_tcase *tcase,
 365                                                                 struct torture_test *test)
 366 {
 367 }
 368
 369 static void harness_test_result (struct torture_context *context,
 370                                                                  enum torture_result res, const char *reason)
 371 {
 372         switch (res) {
 373         case TORTURE_OK:
 374                 printf("ok %s - %s\n", context->active_test->name, reason);
 375                 break;
 376         case TORTURE_FAIL:
 377         case TORTURE_ERROR:
 378                 printf("not ok %s - %s\n", context->active_test->name, reason);
 379                 break;
 380         case TORTURE_SKIP:
 381                 printf("skip %s - %s\n", context->active_test->name, reason);
 382                 break;
 383         }
 384 }
 385
 386 static void harness_comment (struct torture_context *test,
 387                                                          const char *comment)
 388 {
 389         printf("# %s\n", comment);
 390 }
 391
 392 const static struct torture_ui_ops harness_ui_ops = {
 393         .comment = harness_comment,
 394         .test_start = harness_test_start,
 395         .test_result = harness_test_result
 396 };
 397
 398 static void quiet_suite_start(struct torture_context *ctx,
 399                                                   struct torture_suite *suite)
 400 {
 401         int i;
 402         ctx->quiet = true;
 403         for (i = 1; i < ctx->level; i++) putchar('\t');
 404         printf("%s: ", suite->name);
 405         fflush(stdout);
 406 }
 407
 408 static void quiet_suite_finish(struct torture_context *ctx,
 409                                                   struct torture_suite *suite)
 410 {
 411         putchar('\n');
 412 }
 413
 414 static void quiet_test_result (struct torture_context *context,
 415                                                            enum torture_result res, const char *reason)
 416 {
 417         fflush(stdout);
 418         switch (res) {
 419         case TORTURE_OK: putchar('.'); break;
 420         case TORTURE_FAIL: putchar('F'); break;
 421         case TORTURE_ERROR: putchar('E'); break;
 422         case TORTURE_SKIP: putchar('I'); break;
 423         }
 424 }
 425
 426 const static struct torture_ui_ops quiet_ui_ops = {
 427         .suite_start = quiet_suite_start,
 428         .suite_finish = quiet_suite_finish,
 429         .test_result = quiet_test_result
 430 };
 431
 432 void run_recipe(struct torture_context *tctx, const char *recipe)
 433 {
 434         int numlines, i, ret;
 435         char **lines;
 436
 437         lines = file_lines_load(recipe, &numlines, NULL);
 438         if (lines == NULL) {
 439                 fprintf(stderr, "Unable to load file %s\n", recipe);
 440                 return;
 441         }
 442
 443         for (i = 0; i < numlines; i++) {
 444                 int argc;
 445                 const char **argv;
 446
 447                 ret = poptParseArgvString(lines[i], &argc, &argv);
 448                 if (ret != 0) {
 449                         fprintf(stderr, "Error parsing line\n");
 450                         continue;
 451                 }
 452
 453                 run_test(tctx, argv[0]);
 454         }
 455
 456         talloc_free(lines);
 457 }
 458
 459 void run_shell(struct torture_context *tctx)
 460 {
 461         char *cline;
 462         int argc;
 463         const char **argv;
 464         int ret;
 465
 466         while (1) {
 467                 cline = smb_readline("torture> ", NULL, NULL);
 468
 469                 if (cline == NULL)
 470                         return;
 471
 472                 ret = poptParseArgvString(cline, &argc, &argv);
 473                 if (ret != 0) {
 474                         fprintf(stderr, "Error parsing line\n");
 475                         continue;
 476                 }
 477
 478                 if (!strcmp(argv[0], "quit")) {
 479                         return;
 480                 } else if (!strcmp(argv[0], "set")) {
 481                         if (argc < 3) {
 482                                 fprintf(stderr, "Usage: set <variable> <value>\n");
 483                         } else {
 484                                 char *name = talloc_asprintf(NULL, "torture:%s", argv[1]);
 485                                 lp_set_cmdline(name, argv[2]);
 486                                 talloc_free(name);
 487                         }
 488                 } else if (!strcmp(argv[0], "help")) {
 489                         fprintf(stderr, "Available commands:\n"
 490                                                         " help - This help command\n"
 491                                                         " run - Run test\n"
 492                                                         " set - Change variables\n"
 493                                                         "\n");
 494                 } else if (!strcmp(argv[0], "run")) {
 495                         if (argc < 2) {
 496                                 fprintf(stderr, "Usage: run TEST-NAME [OPTIONS...]\n");
 497                         } else {
 498                                 run_test(tctx, argv[1]);
 499                         }
 500                 }
 501         }
 502 }
 503
 504 /****************************************************************************
 505   main program
 506 ****************************************************************************/
 507 int main(int argc,char *argv[])
 508 {
 509         int opt, i;
 510         bool correct = true;
 511         int max_runtime=0;
 512         int argc_new;
 513         struct torture_context *torture;
 514         const struct torture_ui_ops *ui_ops;
 515         char **argv_new;
 516         poptContext pc;
 517         static const char *target = "other";
 518         const char **subunit_dir;
 519         int shell = False;
 520         static const char *ui_ops_name = "simple";
 521         enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
 522               OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC};
 523
 524         struct poptOption long_options[] = {
 525                 POPT_AUTOHELP
 526                 {"format", 0, POPT_ARG_STRING, &ui_ops_name, 0, "Output format (one of: simple, subunit, harness)", NULL },
 527                 {"smb-ports",   'p', POPT_ARG_STRING, NULL,     OPT_SMB_PORTS,  "SMB ports",    NULL},
 528                 {"seed",          0, POPT_ARG_INT,  &torture_seed,      0,      "seed",         NULL},
 529                 {"num-ops",       0, POPT_ARG_INT,  &torture_numops,    0,      "num ops",      NULL},
 530                 {"entries",       0, POPT_ARG_INT,  &torture_entries,   0,      "entries",      NULL},
 531                 {"show-all",      0, POPT_ARG_NONE, &torture_showall,   0,      "show all",     NULL},
 532                 {"loadfile",      0, POPT_ARG_STRING,   NULL,   OPT_LOADFILE,   "loadfile",     NULL},
 533                 {"unclist",       0, POPT_ARG_STRING,   NULL,   OPT_UNCLIST,    "unclist",      NULL},
 534                 {"timelimit",   't', POPT_ARG_STRING,   NULL,   OPT_TIMELIMIT,  "timelimit",    NULL},
 535                 {"failures",    'f', POPT_ARG_INT,  &torture_failures,  0,      "failures",     NULL},
 536                 {"parse-dns",   'D', POPT_ARG_STRING,   NULL,   OPT_DNS,        "parse-dns",    NULL},
 537                 {"dangerous",   'X', POPT_ARG_NONE,     NULL,   OPT_DANGEROUS,
 538                  "run dangerous tests (eg. wiping out password database)", NULL},
 539                 {"shell",               's', POPT_ARG_NONE, &shell, True, "Run shell", NULL},
 540                 {"target",              'T', POPT_ARG_STRING, &target, 0, "samba3|samba4|other", NULL},
 541                 {"async",       'a', POPT_ARG_NONE,     NULL,   OPT_ASYNC,
 542                  "run async tests", NULL},
 543                 {"num-async",    0, POPT_ARG_INT,  &torture_numasync,  0,
 544                  "number of simultaneous async requests", NULL},
 545                 {"maximum-runtime", 0, POPT_ARG_INT, &max_runtime, 0,
 546                  "set maximum time for smbtorture to live", "seconds"},
 547                 POPT_COMMON_SAMBA
 548                 POPT_COMMON_CONNECTION
 549                 POPT_COMMON_CREDENTIALS
 550                 POPT_COMMON_VERSION
 551                 { NULL }
 552         };
 553
 554         setlinebuf(stdout);
 555
 556         /* we are never interested in SIGPIPE */
 557         BlockSignals(true, SIGPIPE);
 558
 559         pc = poptGetContext("smbtorture", argc, (const char **) argv, long_options,
 560                             POPT_CONTEXT_KEEP_FIRST);
 561
 562         poptSetOtherOptionHelp(pc, "<binding>|<unc> TEST1 TEST2 ...");
 563
 564         while((opt = poptGetNextOpt(pc)) != -1) {
 565                 switch (opt) {
 566                 case OPT_LOADFILE:
 567                         lp_set_cmdline("torture:loadfile", poptGetOptArg(pc));
 568                         break;
 569                 case OPT_UNCLIST:
 570                         lp_set_cmdline("torture:unclist", poptGetOptArg(pc));
 571                         break;
 572                 case OPT_TIMELIMIT:
 573                         lp_set_cmdline("torture:timelimit", poptGetOptArg(pc));
 574                         break;
 575                 case OPT_DNS:
 576                         parse_dns(poptGetOptArg(pc));
 577                         break;
 578                 case OPT_DANGEROUS:
 579                         lp_set_cmdline("torture:dangerous", "Yes");
 580                         break;
 581                 case OPT_ASYNC:
 582                         lp_set_cmdline("torture:async", "Yes");
 583                         break;
 584                 case OPT_SMB_PORTS:
 585                         lp_set_cmdline("smb ports", poptGetOptArg(pc));
 586                         break;
 587                 }
 588         }
 589
 590         if (strcmp(target, "samba3") == 0) {
 591                 lp_set_cmdline("torture:samba3", "true");
 592                 lp_set_cmdline("torture:knownfail", "samba3-knownfail");
 593         } else if (strcmp(target, "samba4") == 0) {
 594                 lp_set_cmdline("torture:samba4", "true");
 595                 lp_set_cmdline("torture:knownfail", "samba4-knownfail");
 596         }
 597
 598         if (max_runtime) {
 599                 /* this will only work if nobody else uses alarm(),
 600                    which means it won't work for some tests, but we
 601                    can't use the event context method we use for smbd
 602                    as so many tests create their own event
 603                    context. This will at least catch most cases. */
 604                 signal(SIGALRM, max_runtime_handler);
 605                 alarm(max_runtime);
 606         }
 607
 608         torture_init();
 609         ldb_global_init();
 610
 611         subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
 612         if (subunit_dir == NULL)
 613                 torture_subunit_load_testsuites(dyn_TORTUREDIR, true, NULL);
 614         else {
 615                 for (i = 0; subunit_dir[i]; i++)
 616                         torture_subunit_load_testsuites(subunit_dir[i], true, NULL);
 617         }
 618
 619         if (torture_seed == 0) {
 620                 torture_seed = time(NULL);
 621         }
 622         printf("Using seed %d\n", torture_seed);
 623         srandom(torture_seed);
 624
 625         argv_new = discard_const_p(char *, poptGetArgs(pc));
 626
 627         argc_new = argc;
 628         for (i=0; i<argc; i++) {
 629                 if (argv_new[i] == NULL) {
 630                         argc_new = i;
 631                         break;
 632                 }
 633         }
 634
 635         if (!(argc_new >= 3 || (shell && argc_new >= 2))) {
 636                 usage(pc);
 637                 exit(1);
 638         }
 639
 640         /* see if its a RPC transport specifier */
 641         if (is_binding_string(argv_new[1])) {
 642                 lp_set_cmdline("torture:binding", argv_new[1]);
 643         } else {
 644                 char *binding = NULL;
 645                 char *host = NULL, *share = NULL;
 646
 647                 if (!smbcli_parse_unc(argv_new[1], NULL, &host, &share)) {
 648                         d_printf("Invalid option: %s is not a valid torture target (share or binding string)\n\n", argv_new[1]);
 649                         usage(pc);
 650                 }
 651
 652                 lp_set_cmdline("torture:host", host);
 653                 lp_set_cmdline("torture:share", share);
 654                 asprintf(&binding, "ncacn_np:%s", host);
 655                 lp_set_cmdline("torture:binding", binding);
 656         }
 657
 658         if (!strcmp(ui_ops_name, "simple")) {
 659                 ui_ops = &std_ui_ops;
 660         } else if (!strcmp(ui_ops_name, "subunit")) {
 661                 ui_ops = &subunit_ui_ops;
 662         } else if (!strcmp(ui_ops_name, "harness")) {
 663                 ui_ops = &harness_ui_ops;
 664         } else if (!strcmp(ui_ops_name, "quiet")) {
 665                 ui_ops = &quiet_ui_ops;
 666         } else {
 667                 printf("Unknown output format '%s'\n", ui_ops_name);
 668                 exit(1);
 669         }
 670
 671         torture = torture_context_init(talloc_autofree_context(),
 672                                 lp_parm_string(-1, "torture", "knownfail"), ui_ops);
 673
 674         if (argc_new == 0) {
 675                 printf("You must specify a test to run, or 'ALL'\n");
 676         } else if (shell) {
 677                 run_shell(torture);
 678         } else {
 679                 int total;
 680                 double rate;
 681                 int unexpected_failures;
 682                 for (i=2;i<argc_new;i++) {
 683                         if (argv_new[i][0] == '@') {
 684                                 run_recipe(torture, argv_new[i]+1);
 685                         } else if (!run_test(torture, argv_new[i])) {
 686                                 correct = false;
 687                         }
 688                 }
 689
 690                 unexpected_failures = str_list_length(torture->results.unexpected_failures);
 691
 692                 total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
 693                 if (total == 0) {
 694                         printf("No tests run.\n");
 695                 } else {
 696                         rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
 697
 698                         printf("Tests: %d, Failures: %d", total, torture->results.failed);
 699                         if (torture->results.failed - unexpected_failures) {
 700                                 printf(" (%d expected)", torture->results.failed - unexpected_failures);
 701                         }
 702                         printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
 703                            torture->results.errors, torture->results.skipped, rate);
 704                 }
 705
 706                 if (unexpected_failures) {
 707                         printf("The following tests failed:\n");
 708                         for (i = 0; torture->results.unexpected_failures[i]; i++) {
 709                                 printf("  %s\n", torture->results.unexpected_failures[i]);
 710                         }
 711                         printf("\n");
 712                 }
 713
 714                 if (str_list_length(torture->results.unexpected_errors)) {
 715                         printf("Errors occurred while running the following tests:\n");
 716                         for (i = 0; torture->results.unexpected_errors[i]; i++) {
 717                                 printf("  %s\n", torture->results.unexpected_errors[i]);
 718                         }
 719                         printf("\n");
 720                 }
 721
 722                 if (str_list_length(torture->results.unexpected_successes)) {
 723                         printf("The following tests were expected to fail but succeeded:\n");
 724                         for (i = 0; torture->results.unexpected_successes[i]; i++) {
 725                                 printf("  %s\n", torture->results.unexpected_successes[i]);
 726                         }
 727                         printf("\n");
 728                 }
 729         }
 730
 731         if (torture->results.returncode) {
 732                 return(0);
 733         } else {
 734                 return(1);
 735         }
 736 }