r21028: Generate list of tests rather than maintaining it manually.
[gd/samba-autobuild/.git] / source4 / torture / smbtorture.c
1 /* 
2    Unix SMB/CIFS implementation.
3    SMB torture tester
4    Copyright (C) Andrew Tridgell 1997-2003
5    Copyright (C) Jelmer Vernooij 2006
6    
7    This program is free software; you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 2 of the License, or
10    (at your option) any later version.
11    
12    This program is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16    
17    You should have received a copy of the GNU General Public License
18    along with this program; if not, write to the Free Software
19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 */
21
22 #include "includes.h"
23 #include "lib/cmdline/popt_common.h"
24 #include "system/time.h"
25 #include "system/wait.h"
26 #include "system/filesys.h"
27 #include "system/readline.h"
28 #include "lib/smbreadline/smbreadline.h"
29 #include "libcli/libcli.h"
30 #include "lib/ldb/include/ldb.h"
31 #include "lib/events/events.h"
32 #include "dynconfig.h"
33
34 #include "torture/torture.h"
35 #include "build.h"
36 #include "lib/util/dlinklist.h"
37 #include "librpc/rpc/dcerpc.h"
38
39 static bool run_matching(struct torture_context *torture,
40                                                  const char *prefix, 
41                                                  const char *expr,
42                                                  struct torture_suite *suite,
43                                                  bool *matched)
44 {
45         bool ret = true;
46
47         if (suite == NULL) {
48                 struct torture_suite *o;
49
50                 for (o = torture_root->children; o; o = o->next) {
51                         if (gen_fnmatch(expr, o->name) == 0) {
52                                 *matched = true;
53                                 init_iconv();
54                                 ret &= torture_run_suite(torture, o);
55                                 continue;
56                         }
57
58                         ret &= run_matching(torture, o->name, expr, o, matched);
59                 }
60         } else {
61                 char *name;
62                 struct torture_suite *c;
63                 struct torture_tcase *t;
64
65                 for (c = suite->children; c; c = c->next) {
66                         asprintf(&name, "%s-%s", prefix, c->name);
67
68                         if (gen_fnmatch(expr, name) == 0) {
69                                 *matched = true;
70                                 init_iconv();
71                                 torture->active_testname = talloc_strdup(torture, prefix);
72                                 ret &= torture_run_suite(torture, c);
73                                 free(name);
74                                 continue;
75                         }
76                         
77                         ret &= run_matching(torture, name, expr, c, matched);
78
79                         free(name);
80                 }
81
82                 for (t = suite->testcases; t; t = t->next) {
83                         asprintf(&name, "%s-%s", prefix, t->name);
84                         if (gen_fnmatch(expr, name) == 0) {
85                                 *matched = true;
86                                 init_iconv();
87                                 torture->active_testname = talloc_strdup(torture, prefix);
88                                 ret &= torture_run_tcase(torture, t);
89                                 talloc_free(torture->active_testname);
90                         }
91                         free(name);
92                 }
93         }
94
95         return ret;
96 }
97
98 #define MAX_COLS 80 /* FIXME: Determine this at run-time */
99
100 /****************************************************************************
101 run a specified test or "ALL"
102 ****************************************************************************/
103 static bool run_test(struct torture_context *torture, const char *name)
104 {
105         bool ret = true;
106         bool matched = false;
107         struct torture_suite *o;
108
109         if (strequal(name, "ALL")) {
110                 for (o = torture_root->children; o; o = o->next) {
111                         ret &= torture_run_suite(torture, o);
112                 }
113                 return ret;
114         }
115
116         ret = run_matching(torture, NULL, name, NULL, &matched);
117
118         if (!matched) {
119                 printf("Unknown torture operation '%s'\n", name);
120                 return false;
121         }
122
123         return ret;
124 }
125
126 static void parse_dns(const char *dns)
127 {
128         char *userdn, *basedn, *secret;
129         char *p, *d;
130
131         /* retrievieng the userdn */
132         p = strchr_m(dns, '#');
133         if (!p) {
134                 lp_set_cmdline("torture:ldap_userdn", "");
135                 lp_set_cmdline("torture:ldap_basedn", "");
136                 lp_set_cmdline("torture:ldap_secret", "");
137                 return;
138         }
139         userdn = strndup(dns, p - dns);
140         lp_set_cmdline("torture:ldap_userdn", userdn);
141
142         /* retrieve the basedn */
143         d = p + 1;
144         p = strchr_m(d, '#');
145         if (!p) {
146                 lp_set_cmdline("torture:ldap_basedn", "");
147                 lp_set_cmdline("torture:ldap_secret", "");
148                 return;
149         }
150         basedn = strndup(d, p - d);
151         lp_set_cmdline("torture:ldap_basedn", basedn);
152
153         /* retrieve the secret */
154         p = p + 1;
155         if (!p) {
156                 lp_set_cmdline("torture:ldap_secret", "");
157                 return;
158         }
159         secret = strdup(p);
160         lp_set_cmdline("torture:ldap_secret", secret);
161
162         printf ("%s - %s - %s\n", userdn, basedn, secret);
163
164 }
165
166 static void print_test_list(void)
167 {
168         struct torture_suite *o;
169         struct torture_suite *s;
170         struct torture_tcase *t;
171
172         for (o = torture_root->children; o; o = o->next) {
173                 for (s = o->children; s; s = s->next) {
174                         printf("%s-%s\n", o->name, s->name);
175                 }
176
177                 for (t = o->testcases; t; t = t->next) {
178                         printf("%s-%s\n", o->name, t->name);
179                 }
180         }
181 }
182
183 static void usage(poptContext pc)
184 {
185         struct torture_suite *o;
186         struct torture_suite *s;
187         struct torture_tcase *t;
188         int i;
189
190         poptPrintUsage(pc, stdout, 0);
191         printf("\n");
192
193         printf("The binding format is:\n\n");
194
195         printf("  TRANSPORT:host[flags]\n\n");
196
197         printf("  where TRANSPORT is either ncacn_np for SMB, ncacn_ip_tcp for RPC/TCP\n");
198         printf("  or ncalrpc for local connections.\n\n");
199
200         printf("  'host' is an IP or hostname or netbios name. If the binding string\n");
201         printf("  identifies the server side of an endpoint, 'host' may be an empty\n");
202         printf("  string.\n\n");
203
204         printf("  'flags' can include a SMB pipe name if using the ncacn_np transport or\n");
205         printf("  a TCP port number if using the ncacn_ip_tcp transport, otherwise they\n");
206         printf("  will be auto-determined.\n\n");
207
208         printf("  other recognised flags are:\n\n");
209
210         printf("    sign : enable ntlmssp signing\n");
211         printf("    seal : enable ntlmssp sealing\n");
212         printf("    connect : enable rpc connect level auth (auth, but no sign or seal)\n");
213         printf("    validate: enable the NDR validator\n");
214         printf("    print: enable debugging of the packets\n");
215         printf("    bigendian: use bigendian RPC\n");
216         printf("    padcheck: check reply data for non-zero pad bytes\n\n");
217
218         printf("  For example, these all connect to the samr pipe:\n\n");
219
220         printf("    ncacn_np:myserver\n");
221         printf("    ncacn_np:myserver[samr]\n");
222         printf("    ncacn_np:myserver[\\pipe\\samr]\n");
223         printf("    ncacn_np:myserver[/pipe/samr]\n");
224         printf("    ncacn_np:myserver[samr,sign,print]\n");
225         printf("    ncacn_np:myserver[\\pipe\\samr,sign,seal,bigendian]\n");
226         printf("    ncacn_np:myserver[/pipe/samr,seal,validate]\n");
227         printf("    ncacn_np:\n");
228         printf("    ncacn_np:[/pipe/samr]\n\n");
229
230         printf("    ncacn_ip_tcp:myserver\n");
231         printf("    ncacn_ip_tcp:myserver[1024]\n");
232         printf("    ncacn_ip_tcp:myserver[1024,sign,seal]\n\n");
233
234         printf("    ncalrpc:\n\n");
235
236         printf("The UNC format is:\n\n");
237
238         printf("  //server/share\n\n");
239
240         printf("Tests are:");
241
242         for (o = torture_root->children; o; o = o->next) {
243                 printf("\n%s (%s):\n  ", o->description, o->name);
244
245                 i = 0;
246                 for (s = o->children; s; s = s->next) {
247                         if (i + strlen(o->name) + strlen(s->name) >= (MAX_COLS - 3)) {
248                                 printf("\n  ");
249                                 i = 0;
250                         }
251                         i+=printf("%s-%s ", o->name, s->name);
252                 }
253
254                 for (t = o->testcases; t; t = t->next) {
255                         if (i + strlen(o->name) + strlen(t->name) >= (MAX_COLS - 3)) {
256                                 printf("\n  ");
257                                 i = 0;
258                         }
259                         i+=printf("%s-%s ", o->name, t->name);
260                 }
261
262                 if (i) printf("\n");
263         }
264
265         printf("\nThe default test is ALL.\n");
266
267         exit(1);
268 }
269
270 static void max_runtime_handler(int sig)
271 {
272         DEBUG(0,("maximum runtime exceeded for smbtorture - terminating\n"));
273         exit(1);
274 }
275
276 struct timeval last_suite_started;
277
278 static void simple_suite_start(struct torture_context *ctx,
279                                                            struct torture_suite *suite)
280 {
281         last_suite_started = timeval_current();
282         printf("Running %s\n", suite->name);
283 }
284
285 static void simple_suite_finish(struct torture_context *ctx,
286                                                            struct torture_suite *suite)
287 {
288
289         printf("%s took %g secs\n\n", suite->name, 
290                    timeval_elapsed(&last_suite_started));
291 }
292
293 static void simple_test_result (struct torture_context *context, 
294                                                                 enum torture_result res, const char *reason)
295 {
296         switch (res) {
297         case TORTURE_OK:
298                 if (reason)
299                         printf("OK: %s\n", reason);
300                 break;
301         case TORTURE_FAIL:
302                 printf("TEST %s FAILED! - %s\n", context->active_test->name, reason);
303                 break;
304         case TORTURE_ERROR:
305                 printf("ERROR IN TEST %s! - %s\n", context->active_test->name, reason); 
306                 break;
307         case TORTURE_SKIP:
308                 printf("SKIP: %s - %s\n", context->active_test->name, reason);
309                 break;
310         }
311 }
312
313 static void simple_comment (struct torture_context *test, 
314                                                         const char *comment)
315 {
316         printf("%s", comment);
317 }
318
319 const static struct torture_ui_ops std_ui_ops = {
320         .comment = simple_comment,
321         .suite_start = simple_suite_start,
322         .suite_finish = simple_suite_finish,
323         .test_result = simple_test_result
324 };
325
326 static void subunit_suite_start(struct torture_context *ctx,
327                                                            struct torture_suite *suite)
328 {
329         printf("testsuite: %s\n", suite->path);
330 }
331
332 static void subunit_test_start (struct torture_context *ctx, 
333                                                             struct torture_tcase *tcase,
334                                                                 struct torture_test *test)
335 {
336         printf("test: %s\n", test->name);
337 }
338
339 static void subunit_test_result (struct torture_context *context, 
340                                                                  enum torture_result res, const char *reason)
341 {
342         switch (res) {
343         case TORTURE_OK:
344                 printf("success: %s", context->active_test->name);
345                 break;
346         case TORTURE_FAIL:
347                 printf("failure: %s", context->active_test->name);
348                 break;
349         case TORTURE_ERROR:
350                 printf("error: %s", context->active_test->name);
351                 break;
352         case TORTURE_SKIP:
353                 printf("skip: %s", context->active_test->name);
354                 break;
355         }
356         if (reason)
357                 printf(" [ %s ]", reason);
358         printf("\n");
359 }
360
361 static void subunit_comment (struct torture_context *test, 
362                                                          const char *comment)
363 {
364         fprintf(stderr, "%s", comment);
365 }
366
367 const static struct torture_ui_ops subunit_ui_ops = {
368         .comment = subunit_comment,
369         .test_start = subunit_test_start,
370         .test_result = subunit_test_result,
371         .suite_start = subunit_suite_start
372 };
373
374 static void harness_test_start (struct torture_context *ctx, 
375                                                             struct torture_tcase *tcase,
376                                                                 struct torture_test *test)
377 {
378 }
379
380 static void harness_test_result (struct torture_context *context, 
381                                                                  enum torture_result res, const char *reason)
382 {
383         switch (res) {
384         case TORTURE_OK:
385                 printf("ok %s - %s\n", context->active_test->name, reason);
386                 break;
387         case TORTURE_FAIL:
388         case TORTURE_ERROR:
389                 printf("not ok %s - %s\n", context->active_test->name, reason);
390                 break;
391         case TORTURE_SKIP:
392                 printf("skip %s - %s\n", context->active_test->name, reason);
393                 break;
394         }
395 }
396
397 static void harness_comment (struct torture_context *test, 
398                                                          const char *comment)
399 {
400         printf("# %s\n", comment);
401 }
402
403 const static struct torture_ui_ops harness_ui_ops = {
404         .comment = harness_comment,
405         .test_start = harness_test_start,
406         .test_result = harness_test_result
407 };
408
409 static void quiet_suite_start(struct torture_context *ctx,
410                                                   struct torture_suite *suite)
411 {
412         int i;
413         ctx->quiet = true;
414         for (i = 1; i < ctx->level; i++) putchar('\t');
415         printf("%s: ", suite->name);
416         fflush(stdout);
417 }
418
419 static void quiet_suite_finish(struct torture_context *ctx,
420                                                   struct torture_suite *suite)
421 {
422         putchar('\n');
423 }
424
425 static void quiet_test_result (struct torture_context *context, 
426                                                            enum torture_result res, const char *reason)
427 {
428         fflush(stdout);
429         switch (res) {
430         case TORTURE_OK: putchar('.'); break;
431         case TORTURE_FAIL: putchar('F'); break;
432         case TORTURE_ERROR: putchar('E'); break;
433         case TORTURE_SKIP: putchar('I'); break;
434         }
435 }
436
437 const static struct torture_ui_ops quiet_ui_ops = {
438         .suite_start = quiet_suite_start,
439         .suite_finish = quiet_suite_finish,
440         .test_result = quiet_test_result
441 };
442
443 void run_shell(struct torture_context *tctx)
444 {
445         char *cline;
446         int argc;
447         const char **argv;
448         int ret;
449
450         while (1) {
451                 cline = smb_readline("torture> ", NULL, NULL);
452
453                 if (cline == NULL)
454                         return;
455         
456                 ret = poptParseArgvString(cline, &argc, &argv);
457                 if (ret != 0) {
458                         fprintf(stderr, "Error parsing line\n");
459                         continue;
460                 }
461
462                 if (!strcmp(argv[0], "quit")) {
463                         return;
464                 } else if (!strcmp(argv[0], "set")) {
465                         if (argc < 3) {
466                                 fprintf(stderr, "Usage: set <variable> <value>\n");
467                         } else {
468                                 char *name = talloc_asprintf(NULL, "torture:%s", argv[1]);
469                                 lp_set_cmdline(name, argv[2]);
470                                 talloc_free(name);
471                         }
472                 } else if (!strcmp(argv[0], "help")) {
473                         fprintf(stderr, "Available commands:\n"
474                                                         " help - This help command\n"
475                                                         " run - Run test\n"
476                                                         " set - Change variables\n"
477                                                         "\n");
478                 } else if (!strcmp(argv[0], "run")) {
479                         if (argc < 2) {
480                                 fprintf(stderr, "Usage: run TEST-NAME [OPTIONS...]\n");
481                         } else {
482                                 run_test(tctx, argv[1]);
483                         }
484                 }
485         }
486 }
487
488 /****************************************************************************
489   main program
490 ****************************************************************************/
491 int main(int argc,char *argv[])
492 {
493         int opt, i;
494         bool correct = true;
495         int max_runtime=0;
496         int argc_new;
497         struct torture_context *torture;
498         const struct torture_ui_ops *ui_ops;
499         char **argv_new;
500         poptContext pc;
501         static const char *target = "other";
502         const char **subunit_dir;
503         struct dcerpc_binding *binding_struct;
504         NTSTATUS status;
505         int shell = False;
506         static const char *ui_ops_name = "simple";
507         static int list_tests = 0;
508         enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS, OPT_LIST,
509               OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC,OPT_NUMPROGS, OPT_BASEDIR};
510         
511         struct poptOption long_options[] = {
512                 POPT_AUTOHELP
513                 {"format", 0, POPT_ARG_STRING, &ui_ops_name, 0, "Output format (one of: simple, subunit, harness)", NULL },
514                 {"smb-ports",   'p', POPT_ARG_STRING, NULL,     OPT_SMB_PORTS,  "SMB ports",    NULL},
515                 {"basedir",       0, POPT_ARG_STRING, NULL, OPT_BASEDIR, "base directory", "BSAEDIR" },
516                 {"seed",          0, POPT_ARG_INT,  &torture_seed,      0,      "seed",         NULL},
517                 {"num-progs",     0, POPT_ARG_INT,  NULL,       OPT_NUMPROGS,   "num progs",    NULL},
518                 {"num-ops",       0, POPT_ARG_INT,  &torture_numops,    0,      "num ops",      NULL},
519                 {"entries",       0, POPT_ARG_INT,  &torture_entries,   0,      "entries",      NULL},
520                 {"loadfile",      0, POPT_ARG_STRING,   NULL,   OPT_LOADFILE,   "loadfile",     NULL},
521                 {"list",          0, POPT_ARG_NONE, &list_tests, 0, NULL, NULL },
522                 {"unclist",       0, POPT_ARG_STRING,   NULL,   OPT_UNCLIST,    "unclist",      NULL},
523                 {"timelimit",   't', POPT_ARG_INT,      NULL,   OPT_TIMELIMIT,  "timelimit",    NULL},
524                 {"failures",    'f', POPT_ARG_INT,  &torture_failures,  0,      "failures",     NULL},
525                 {"parse-dns",   'D', POPT_ARG_STRING,   NULL,   OPT_DNS,        "parse-dns",    NULL},
526                 {"dangerous",   'X', POPT_ARG_NONE,     NULL,   OPT_DANGEROUS,
527                  "run dangerous tests (eg. wiping out password database)", NULL},
528                 {"shell",               0, POPT_ARG_NONE, &shell, True, "Run shell", NULL},
529                 {"target",              'T', POPT_ARG_STRING, &target, 0, "samba3|samba4|other", NULL},
530                 {"async",       'a', POPT_ARG_NONE,     NULL,   OPT_ASYNC,
531                  "run async tests", NULL},
532                 {"num-async",    0, POPT_ARG_INT,  &torture_numasync,  0,
533                  "number of simultaneous async requests", NULL},
534                 {"maximum-runtime", 0, POPT_ARG_INT, &max_runtime, 0, 
535                  "set maximum time for smbtorture to live", "seconds"},
536                 POPT_COMMON_SAMBA
537                 POPT_COMMON_CONNECTION
538                 POPT_COMMON_CREDENTIALS
539                 POPT_COMMON_VERSION
540                 { NULL }
541         };
542
543         setlinebuf(stdout);
544
545         /* we are never interested in SIGPIPE */
546         BlockSignals(true, SIGPIPE);
547
548         pc = poptGetContext("smbtorture", argc, (const char **) argv, long_options, 
549                             POPT_CONTEXT_KEEP_FIRST);
550
551         poptSetOtherOptionHelp(pc, "<binding>|<unc> TEST1 TEST2 ...");
552
553         while((opt = poptGetNextOpt(pc)) != -1) {
554                 switch (opt) {
555                 case OPT_LOADFILE:
556                         lp_set_cmdline("torture:loadfile", poptGetOptArg(pc));
557                         break;
558                 case OPT_UNCLIST:
559                         lp_set_cmdline("torture:unclist", poptGetOptArg(pc));
560                         break;
561                 case OPT_TIMELIMIT:
562                         lp_set_cmdline("torture:timelimit", poptGetOptArg(pc));
563                         break;
564                 case OPT_NUMPROGS:
565                         lp_set_cmdline("torture:nprocs", poptGetOptArg(pc));
566                         break;
567                 case OPT_BASEDIR:
568                         lp_set_cmdline("torture:basedir", poptGetOptArg(pc));
569                         break;
570                 case OPT_DNS:
571                         parse_dns(poptGetOptArg(pc));
572                         break;
573                 case OPT_DANGEROUS:
574                         lp_set_cmdline("torture:dangerous", "Yes");
575                         break;
576                 case OPT_ASYNC:
577                         lp_set_cmdline("torture:async", "Yes");
578                         break;
579                 case OPT_SMB_PORTS:
580                         lp_set_cmdline("smb ports", poptGetOptArg(pc));
581                         break;
582                 }
583         }
584
585         if (strcmp(target, "samba3") == 0) {
586                 lp_set_cmdline("torture:samba3", "true");
587                 lp_set_cmdline("torture:knownfail", "samba3-knownfail");
588         } else if (strcmp(target, "samba4") == 0) {
589                 lp_set_cmdline("torture:samba4", "true");
590                 lp_set_cmdline("torture:knownfail", "samba4-knownfail");
591         }
592
593         if (max_runtime) {
594                 /* this will only work if nobody else uses alarm(),
595                    which means it won't work for some tests, but we
596                    can't use the event context method we use for smbd
597                    as so many tests create their own event
598                    context. This will at least catch most cases. */
599                 signal(SIGALRM, max_runtime_handler);
600                 alarm(max_runtime);
601         }
602
603         torture_init();
604         ldb_global_init();
605
606         if (list_tests) {
607                 print_test_list();
608                 return 0;
609         }
610
611         subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
612         if (subunit_dir == NULL) 
613                 torture_subunit_load_testsuites(dyn_TORTUREDIR, true, NULL);
614         else {
615                 for (i = 0; subunit_dir[i]; i++)
616                         torture_subunit_load_testsuites(subunit_dir[i], true, NULL);
617         }
618
619         if (torture_seed == 0) {
620                 torture_seed = time(NULL);
621         } 
622         printf("Using seed %d\n", torture_seed);
623         srandom(torture_seed);
624
625         argv_new = discard_const_p(char *, poptGetArgs(pc));
626
627         argc_new = argc;
628         for (i=0; i<argc; i++) {
629                 if (argv_new[i] == NULL) {
630                         argc_new = i;
631                         break;
632                 }
633         }
634
635         if (!(argc_new >= 3 || (shell && argc_new >= 2))) {
636                 usage(pc);
637                 exit(1);
638         }
639
640         /* see if its a RPC transport specifier */
641         status = dcerpc_parse_binding(talloc_autofree_context(), argv_new[1], &binding_struct);
642         if (NT_STATUS_IS_OK(status)) {
643                 lp_set_cmdline("torture:host", binding_struct->host);
644                 lp_set_cmdline("torture:share", "IPC$");
645                 lp_set_cmdline("torture:binding", argv_new[1]);
646         } else {
647                 char *binding = NULL;
648                 char *host = NULL, *share = NULL;
649
650                 if (!smbcli_parse_unc(argv_new[1], NULL, &host, &share)) {
651                         d_printf("Invalid option: %s is not a valid torture target (share or binding string)\n\n", argv_new[1]);
652                         usage(pc);
653                 }
654
655                 lp_set_cmdline("torture:host", host);
656                 lp_set_cmdline("torture:share", share);
657                 asprintf(&binding, "ncacn_np:%s", host);
658                 lp_set_cmdline("torture:binding", binding);
659         }
660
661         if (!strcmp(ui_ops_name, "simple")) {
662                 ui_ops = &std_ui_ops;
663         } else if (!strcmp(ui_ops_name, "subunit")) {
664                 ui_ops = &subunit_ui_ops;
665         } else if (!strcmp(ui_ops_name, "harness")) {
666                 ui_ops = &harness_ui_ops;
667         } else if (!strcmp(ui_ops_name, "quiet")) {
668                 ui_ops = &quiet_ui_ops;
669         } else {
670                 printf("Unknown output format '%s'\n", ui_ops_name);
671                 exit(1);
672         }
673
674         torture = torture_context_init(talloc_autofree_context(), 
675                                 lp_parm_string(-1, "torture", "knownfail"), ui_ops);
676
677         if (argc_new == 0) {
678                 printf("You must specify a test to run, or 'ALL'\n");
679         } else if (shell) {
680                 run_shell(torture);
681         } else {
682                 int total;
683                 double rate;
684                 int unexpected_failures;
685                 for (i=2;i<argc_new;i++) {
686                         if (!run_test(torture, argv_new[i])) {
687                                 correct = false;
688                         }
689                 }
690
691                 unexpected_failures = str_list_length(torture->results.unexpected_failures);
692
693                 total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
694                 if (total == 0) {
695                         printf("No tests run.\n");
696                 } else {
697                         rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
698                 
699                         printf("Tests: %d, Failures: %d", total, torture->results.failed);
700                         if (torture->results.failed - unexpected_failures) {
701                                 printf(" (%d expected)", torture->results.failed - unexpected_failures);
702                         }
703                         printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
704                            torture->results.errors, torture->results.skipped, rate);
705                 }
706
707                 if (unexpected_failures) {
708                         printf("The following tests failed:\n");
709                         for (i = 0; torture->results.unexpected_failures[i]; i++) {
710                                 printf("  %s\n", torture->results.unexpected_failures[i]);
711                         }
712                         printf("\n");
713                 }
714
715                 if (str_list_length(torture->results.unexpected_errors)) {
716                         printf("Errors occurred while running the following tests:\n");
717                         for (i = 0; torture->results.unexpected_errors[i]; i++) {
718                                 printf("  %s\n", torture->results.unexpected_errors[i]);
719                         }
720                         printf("\n");
721                 }
722
723                 if (str_list_length(torture->results.unexpected_successes)) {
724                         printf("The following tests were expected to fail but succeeded:\n");
725                         for (i = 0; torture->results.unexpected_successes[i]; i++) {
726                                 printf("  %s\n", torture->results.unexpected_successes[i]);
727                         }
728                         printf("\n");
729                 }
730         }
731
732         if (torture->results.returncode) {
733                 return(0);
734         } else {
735                 return(1);
736         }
737 }