r19381: Print out the specific tests that failed after a smbtorture run.
[gd/samba-autobuild/.git] / source4 / torture / smbtorture.c
1 /* 
2    Unix SMB/CIFS implementation.
3    SMB torture tester
4    Copyright (C) Andrew Tridgell 1997-2003
5    Copyright (C) Jelmer Vernooij 2006
6    
7    This program is free software; you can redistribute it and/or modify
8    it under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 2 of the License, or
10    (at your option) any later version.
11    
12    This program is distributed in the hope that it will be useful,
13    but WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15    GNU General Public License for more details.
16    
17    You should have received a copy of the GNU General Public License
18    along with this program; if not, write to the Free Software
19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 */
21
22 #include "includes.h"
23 #include "lib/cmdline/popt_common.h"
24 #include "system/time.h"
25 #include "system/wait.h"
26 #include "system/filesys.h"
27 #include "libcli/libcli.h"
28 #include "lib/ldb/include/ldb.h"
29 #include "lib/events/events.h"
30 #include "dynconfig.h"
31
32 #include "torture/torture.h"
33 #include "build.h"
34 #include "lib/util/dlinklist.h"
35 #include "librpc/rpc/dcerpc.h"
36
37 static bool run_matching(struct torture_context *torture,
38                                                  const char *prefix, 
39                                                  const char *expr,
40                                                  struct torture_suite *suite,
41                                                  bool *matched)
42 {
43         bool ret = true;
44
45         if (suite == NULL) {
46                 struct torture_suite *o;
47
48                 for (o = torture_root->children; o; o = o->next) {
49                         if (gen_fnmatch(expr, o->name) == 0) {
50                                 *matched = true;
51                                 init_iconv();
52                                 ret &= torture_run_suite(torture, o);
53                                 continue;
54                         }
55
56                         ret &= run_matching(torture, o->name, expr, o, matched);
57                 }
58         } else {
59                 char *name;
60                 struct torture_suite *c;
61                 struct torture_tcase *t;
62
63                 for (c = suite->children; c; c = c->next) {
64                         asprintf(&name, "%s-%s", prefix, c->name);
65                         if (gen_fnmatch(expr, name) == 0) {
66                                 *matched = true;
67                                 init_iconv();
68                                 ret &= torture_run_suite(torture, c);
69                                 free(name);
70                                 continue;
71                         }
72                         
73                         ret &= run_matching(torture, name, expr, c, matched);
74
75                         free(name);
76                 }
77
78                 for (t = suite->testcases; t; t = t->next) {
79                         asprintf(&name, "%s-%s", prefix, t->name);
80                         if (gen_fnmatch(expr, name) == 0) {
81                                 *matched = true;
82                                 init_iconv();
83                                 ret &= torture_run_tcase(torture, t);
84                         }
85                         free(name);
86                 }
87         }
88
89         return ret;
90 }
91
92 #define MAX_COLS 80 /* FIXME: Determine this at run-time */
93
94 /****************************************************************************
95 run a specified test or "ALL"
96 ****************************************************************************/
97 static bool run_test(struct torture_context *torture, const char *name)
98 {
99         bool ret = true;
100         bool matched = false;
101         struct torture_suite *o;
102
103         if (strequal(name, "ALL")) {
104                 for (o = torture_root->children; o; o = o->next) {
105                         ret &= torture_run_suite(torture, o);
106                 }
107                 return ret;
108         }
109
110         ret = run_matching(torture, NULL, name, NULL, &matched);
111
112         if (!matched) {
113                 printf("Unknown torture operation '%s'\n", name);
114                 return false;
115         }
116
117         return ret;
118 }
119
120 static void parse_dns(const char *dns)
121 {
122         char *userdn, *basedn, *secret;
123         char *p, *d;
124
125         /* retrievieng the userdn */
126         p = strchr_m(dns, '#');
127         if (!p) {
128                 lp_set_cmdline("torture:ldap_userdn", "");
129                 lp_set_cmdline("torture:ldap_basedn", "");
130                 lp_set_cmdline("torture:ldap_secret", "");
131                 return;
132         }
133         userdn = strndup(dns, p - dns);
134         lp_set_cmdline("torture:ldap_userdn", userdn);
135
136         /* retrieve the basedn */
137         d = p + 1;
138         p = strchr_m(d, '#');
139         if (!p) {
140                 lp_set_cmdline("torture:ldap_basedn", "");
141                 lp_set_cmdline("torture:ldap_secret", "");
142                 return;
143         }
144         basedn = strndup(d, p - d);
145         lp_set_cmdline("torture:ldap_basedn", basedn);
146
147         /* retrieve the secret */
148         p = p + 1;
149         if (!p) {
150                 lp_set_cmdline("torture:ldap_secret", "");
151                 return;
152         }
153         secret = strdup(p);
154         lp_set_cmdline("torture:ldap_secret", secret);
155
156         printf ("%s - %s - %s\n", userdn, basedn, secret);
157
158 }
159
160 static void usage(poptContext pc)
161 {
162         struct torture_suite *o;
163         struct torture_suite *s;
164         struct torture_tcase *t;
165         int i;
166
167         poptPrintUsage(pc, stdout, 0);
168         printf("\n");
169
170         printf("The binding format is:\n\n");
171
172         printf("  TRANSPORT:host[flags]\n\n");
173
174         printf("  where TRANSPORT is either ncacn_np for SMB, ncacn_ip_tcp for RPC/TCP\n");
175         printf("  or ncalrpc for local connections.\n\n");
176
177         printf("  'host' is an IP or hostname or netbios name. If the binding string\n");
178         printf("  identifies the server side of an endpoint, 'host' may be an empty\n");
179         printf("  string.\n\n");
180
181         printf("  'flags' can include a SMB pipe name if using the ncacn_np transport or\n");
182         printf("  a TCP port number if using the ncacn_ip_tcp transport, otherwise they\n");
183         printf("  will be auto-determined.\n\n");
184
185         printf("  other recognised flags are:\n\n");
186
187         printf("    sign : enable ntlmssp signing\n");
188         printf("    seal : enable ntlmssp sealing\n");
189         printf("    connect : enable rpc connect level auth (auth, but no sign or seal)\n");
190         printf("    validate: enable the NDR validator\n");
191         printf("    print: enable debugging of the packets\n");
192         printf("    bigendian: use bigendian RPC\n");
193         printf("    padcheck: check reply data for non-zero pad bytes\n\n");
194
195         printf("  For example, these all connect to the samr pipe:\n\n");
196
197         printf("    ncacn_np:myserver\n");
198         printf("    ncacn_np:myserver[samr]\n");
199         printf("    ncacn_np:myserver[\\pipe\\samr]\n");
200         printf("    ncacn_np:myserver[/pipe/samr]\n");
201         printf("    ncacn_np:myserver[samr,sign,print]\n");
202         printf("    ncacn_np:myserver[\\pipe\\samr,sign,seal,bigendian]\n");
203         printf("    ncacn_np:myserver[/pipe/samr,seal,validate]\n");
204         printf("    ncacn_np:\n");
205         printf("    ncacn_np:[/pipe/samr]\n\n");
206
207         printf("    ncacn_ip_tcp:myserver\n");
208         printf("    ncacn_ip_tcp:myserver[1024]\n");
209         printf("    ncacn_ip_tcp:myserver[1024,sign,seal]\n\n");
210
211         printf("    ncalrpc:\n\n");
212
213         printf("The UNC format is:\n\n");
214
215         printf("  //server/share\n\n");
216
217         printf("Tests are:");
218
219         for (o = torture_root->children; o; o = o->next) {
220                 printf("\n%s (%s):\n  ", o->description, o->name);
221
222                 i = 0;
223                 for (s = o->children; s; s = s->next) {
224                         if (i + strlen(o->name) + strlen(s->name) >= (MAX_COLS - 3)) {
225                                 printf("\n  ");
226                                 i = 0;
227                         }
228                         i+=printf("%s-%s ", o->name, s->name);
229                 }
230
231                 for (t = o->testcases; t; t = t->next) {
232                         if (i + strlen(o->name) + strlen(t->name) >= (MAX_COLS - 3)) {
233                                 printf("\n  ");
234                                 i = 0;
235                         }
236                         i+=printf("%s-%s ", o->name, t->name);
237                 }
238
239                 if (i) printf("\n");
240         }
241
242         printf("\nThe default test is ALL.\n");
243
244         exit(1);
245 }
246
247 static bool is_binding_string(const char *binding_string)
248 {
249         TALLOC_CTX *mem_ctx = talloc_named_const(NULL, 0, "is_binding_string");
250         struct dcerpc_binding *binding_struct;
251         NTSTATUS status;
252         
253         status = dcerpc_parse_binding(mem_ctx, binding_string, &binding_struct);
254
255         talloc_free(mem_ctx);
256         return NT_STATUS_IS_OK(status);
257 }
258
259 static void max_runtime_handler(int sig)
260 {
261         DEBUG(0,("maximum runtime exceeded for smbtorture - terminating\n"));
262         exit(1);
263 }
264
265 struct timeval last_suite_started;
266
267 static void simple_suite_start(struct torture_context *ctx,
268                                                            struct torture_suite *suite)
269 {
270         last_suite_started = timeval_current();
271         printf("Running %s\n", suite->name);
272 }
273
274 static void simple_suite_finish(struct torture_context *ctx,
275                                                            struct torture_suite *suite)
276 {
277
278         printf("%s took %g secs\n\n", suite->name, 
279                    timeval_elapsed(&last_suite_started));
280 }
281
282 static void simple_test_result (struct torture_context *context, 
283                                                                 enum torture_result res, const char *reason)
284 {
285         switch (res) {
286         case TORTURE_OK:
287                 if (reason)
288                         printf("OK: %s\n", reason);
289                 break;
290         case TORTURE_FAIL:
291                 printf("TEST %s FAILED! - %s\n", context->active_test->name, reason);
292                 break;
293         case TORTURE_ERROR:
294                 printf("ERROR IN TEST %s! - %s\n", context->active_test->name, reason); 
295                 break;
296         case TORTURE_SKIP:
297                 printf("SKIP: %s - %s\n", context->active_test->name, reason);
298                 break;
299         }
300 }
301
302 static void simple_comment (struct torture_context *test, 
303                                                         const char *comment)
304 {
305         printf("%s", comment);
306 }
307
308 const static struct torture_ui_ops std_ui_ops = {
309         .comment = simple_comment,
310         .suite_start = simple_suite_start,
311         .suite_finish = simple_suite_finish,
312         .test_result = simple_test_result
313 };
314
315
316 static void subunit_test_start (struct torture_context *ctx, 
317                                                             struct torture_tcase *tcase,
318                                                                 struct torture_test *test)
319 {
320         printf("test: %s\n", test->name);
321 }
322
323 static void subunit_test_result (struct torture_context *context, 
324                                                                  enum torture_result res, const char *reason)
325 {
326         switch (res) {
327         case TORTURE_OK:
328                 printf("success: %s", context->active_test->name);
329                 break;
330         case TORTURE_FAIL:
331                 printf("failure: %s", context->active_test->name);
332                 break;
333         case TORTURE_ERROR:
334                 printf("error: %s", context->active_test->name);
335                 break;
336         case TORTURE_SKIP:
337                 printf("skip: %s", context->active_test->name);
338                 break;
339         }
340         if (reason)
341                 printf(" [ %s ]", reason);
342         printf("\n");
343 }
344
345 static void subunit_comment (struct torture_context *test, 
346                                                          const char *comment)
347 {
348         fprintf(stderr, "%s", comment);
349 }
350
351 const static struct torture_ui_ops subunit_ui_ops = {
352         .comment = subunit_comment,
353         .test_start = subunit_test_start,
354         .test_result = subunit_test_result
355 };
356
357 static void harness_test_start (struct torture_context *ctx, 
358                                                             struct torture_tcase *tcase,
359                                                                 struct torture_test *test)
360 {
361 }
362
363 static void harness_test_result (struct torture_context *context, 
364                                                                  enum torture_result res, const char *reason)
365 {
366         switch (res) {
367         case TORTURE_OK:
368                 printf("ok %s - %s\n", context->active_test->name, reason);
369                 break;
370         case TORTURE_FAIL:
371         case TORTURE_ERROR:
372                 printf("not ok %s - %s\n", context->active_test->name, reason);
373                 break;
374         case TORTURE_SKIP:
375                 printf("skip %s - %s\n", context->active_test->name, reason);
376                 break;
377         }
378 }
379
380 static void harness_comment (struct torture_context *test, 
381                                                          const char *comment)
382 {
383         printf("# %s\n", comment);
384 }
385
386 const static struct torture_ui_ops harness_ui_ops = {
387         .comment = harness_comment,
388         .test_start = harness_test_start,
389         .test_result = harness_test_result
390 };
391
392 static void quiet_suite_start(struct torture_context *ctx,
393                                                   struct torture_suite *suite)
394 {
395         int i;
396         ctx->quiet = true;
397         for (i = 1; i < ctx->level; i++) putchar('\t');
398         printf("%s: ", suite->name);
399         fflush(stdout);
400 }
401
402 static void quiet_suite_finish(struct torture_context *ctx,
403                                                   struct torture_suite *suite)
404 {
405         putchar('\n');
406 }
407
408 static void quiet_test_result (struct torture_context *context, 
409                                                            enum torture_result res, const char *reason)
410 {
411         fflush(stdout);
412         switch (res) {
413         case TORTURE_OK: putchar('.'); break;
414         case TORTURE_FAIL: putchar('F'); break;
415         case TORTURE_ERROR: putchar('E'); break;
416         case TORTURE_SKIP: putchar('I'); break;
417         }
418 }
419
420 const static struct torture_ui_ops quiet_ui_ops = {
421         .suite_start = quiet_suite_start,
422         .suite_finish = quiet_suite_finish,
423         .test_result = quiet_test_result
424 };
425
426
427 /****************************************************************************
428   main program
429 ****************************************************************************/
430  int main(int argc,char *argv[])
431 {
432         int opt, i;
433         bool correct = true;
434         int max_runtime=0;
435         int argc_new;
436         struct torture_context *torture;
437         const struct torture_ui_ops *ui_ops;
438         char **argv_new;
439         poptContext pc;
440         static const char *target = "other";
441         const char **subunit_dir;
442         static const char *ui_ops_name = "simple";
443         enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
444               OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC};
445         
446         struct poptOption long_options[] = {
447                 POPT_AUTOHELP
448                 {"format", 0, POPT_ARG_STRING, &ui_ops_name, 0, "Output format (one of: simple, subunit, harness)", NULL },
449                 {"smb-ports",   'p', POPT_ARG_STRING, NULL,     OPT_SMB_PORTS,  "SMB ports",    NULL},
450                 {"seed",          0, POPT_ARG_INT,  &torture_seed,      0,      "seed",         NULL},
451                 {"num-progs",     0, POPT_ARG_INT,  &torture_nprocs,    0,      "num progs",    NULL},
452                 {"num-ops",       0, POPT_ARG_INT,  &torture_numops,    0,      "num ops",      NULL},
453                 {"entries",       0, POPT_ARG_INT,  &torture_entries,   0,      "entries",      NULL},
454                 {"show-all",      0, POPT_ARG_NONE, &torture_showall,   0,      "show all",     NULL},
455                 {"loadfile",      0, POPT_ARG_STRING,   NULL,   OPT_LOADFILE,   "loadfile",     NULL},
456                 {"unclist",       0, POPT_ARG_STRING,   NULL,   OPT_UNCLIST,    "unclist",      NULL},
457                 {"timelimit",   't', POPT_ARG_STRING,   NULL,   OPT_TIMELIMIT,  "timelimit",    NULL},
458                 {"failures",    'f', POPT_ARG_INT,  &torture_failures,  0,      "failures",     NULL},
459                 {"parse-dns",   'D', POPT_ARG_STRING,   NULL,   OPT_DNS,        "parse-dns",    NULL},
460                 {"dangerous",   'X', POPT_ARG_NONE,     NULL,   OPT_DANGEROUS,
461                  "run dangerous tests (eg. wiping out password database)", NULL},
462                 {"target",              'T', POPT_ARG_STRING, &target, 0, "samba3|samba4|other", NULL},
463                 {"async",       'a', POPT_ARG_NONE,     NULL,   OPT_ASYNC,
464                  "run async tests", NULL},
465                 {"num-async",    0, POPT_ARG_INT,  &torture_numasync,  0,
466                  "number of simultaneous async requests", NULL},
467                 {"maximum-runtime", 0, POPT_ARG_INT, &max_runtime, 0, 
468                  "set maximum time for smbtorture to live", "seconds"},
469                 POPT_COMMON_SAMBA
470                 POPT_COMMON_CONNECTION
471                 POPT_COMMON_CREDENTIALS
472                 POPT_COMMON_VERSION
473                 { NULL }
474         };
475
476         setlinebuf(stdout);
477
478         /* we are never interested in SIGPIPE */
479         BlockSignals(true,SIGPIPE);
480
481         pc = poptGetContext("smbtorture", argc, (const char **) argv, long_options, 
482                             POPT_CONTEXT_KEEP_FIRST);
483
484         poptSetOtherOptionHelp(pc, "<binding>|<unc> TEST1 TEST2 ...");
485
486         while((opt = poptGetNextOpt(pc)) != -1) {
487                 switch (opt) {
488                 case OPT_LOADFILE:
489                         lp_set_cmdline("torture:loadfile", poptGetOptArg(pc));
490                         break;
491                 case OPT_UNCLIST:
492                         lp_set_cmdline("torture:unclist", poptGetOptArg(pc));
493                         break;
494                 case OPT_TIMELIMIT:
495                         lp_set_cmdline("torture:timelimit", poptGetOptArg(pc));
496                         break;
497                 case OPT_DNS:
498                         parse_dns(poptGetOptArg(pc));
499                         break;
500                 case OPT_DANGEROUS:
501                         lp_set_cmdline("torture:dangerous", "Yes");
502                         break;
503                 case OPT_ASYNC:
504                         lp_set_cmdline("torture:async", "Yes");
505                         break;
506                 case OPT_SMB_PORTS:
507                         lp_set_cmdline("smb ports", poptGetOptArg(pc));
508                         break;
509                 default:
510                         d_printf("Invalid option %s: %s\n", 
511                                  poptBadOption(pc, 0), poptStrerror(opt));
512                         torture_init();
513                         usage(pc);
514                         exit(1);
515                 }
516         }
517
518         if (strcmp(target, "samba3") == 0) {
519                 lp_set_cmdline("target:samba3", "true");
520         } else if (strcmp(target, "samba4") == 0) {
521                 lp_set_cmdline("target:samba4", "true");
522         }
523
524         if (max_runtime) {
525                 /* this will only work if nobody else uses alarm(),
526                    which means it won't work for some tests, but we
527                    can't use the event context method we use for smbd
528                    as so many tests create their own event
529                    context. This will at least catch most cases. */
530                 signal(SIGALRM, max_runtime_handler);
531                 alarm(max_runtime);
532         }
533
534         torture_init();
535         ldb_global_init();
536
537         subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
538         if (subunit_dir == NULL) 
539                 torture_subunit_load_testsuites(dyn_TORTUREDIR, true, NULL);
540         else {
541                 for (i = 0; subunit_dir[i]; i++)
542                         torture_subunit_load_testsuites(subunit_dir[i], true, NULL);
543         }
544
545         if (torture_seed == 0) {
546                 torture_seed = time(NULL);
547         } 
548         printf("Using seed %d\n", torture_seed);
549         srandom(torture_seed);
550
551         argv_new = discard_const_p(char *, poptGetArgs(pc));
552
553         argc_new = argc;
554         for (i=0; i<argc; i++) {
555                 if (argv_new[i] == NULL) {
556                         argc_new = i;
557                         break;
558                 }
559         }
560
561         if (argc_new < 3) {
562                 usage(pc);
563                 exit(1);
564         }
565
566         /* see if its a RPC transport specifier */
567         if (is_binding_string(argv_new[1])) {
568                 lp_set_cmdline("torture:binding", argv_new[1]);
569         } else {
570                 char *binding = NULL;
571                 char *host = NULL, *share = NULL;
572
573                 if (!smbcli_parse_unc(argv_new[1], NULL, &host, &share)) {
574                         d_printf("Invalid option: %s is not a valid torture target (share or binding string)\n\n", argv_new[1]);
575                         usage(pc);
576                 }
577
578                 lp_set_cmdline("torture:host", host);
579                 lp_set_cmdline("torture:share", share);
580                 asprintf(&binding, "ncacn_np:%s", host);
581                 lp_set_cmdline("torture:binding", binding);
582         }
583
584         if (!strcmp(ui_ops_name, "simple")) {
585                 ui_ops = &std_ui_ops;
586         } else if (!strcmp(ui_ops_name, "subunit")) {
587                 ui_ops = &subunit_ui_ops;
588         } else if (!strcmp(ui_ops_name, "harness")) {
589                 ui_ops = &harness_ui_ops;
590         } else if (!strcmp(ui_ops_name, "quiet")) {
591                 ui_ops = &quiet_ui_ops;
592         } else {
593                 printf("Unknown output format '%s'\n", ui_ops_name);
594                 exit(1);
595         }
596
597         torture = torture_context_init(talloc_autofree_context(), "KNOWN_FAILURES", 
598                                                  ui_ops);
599
600         if (argc_new == 0) {
601                 printf("You must specify a test to run, or 'ALL'\n");
602         } else {
603                 int total;
604                 double rate;
605                 int unexpected_failures;
606                 for (i=2;i<argc_new;i++) {
607                         if (!run_test(torture, argv_new[i])) {
608                                 correct = false;
609                         }
610                 }
611
612
613                 unexpected_failures = str_list_length(torture->results.unexpected_failures);
614
615                 total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
616                 if (total == 0) {
617                         printf("No tests run.\n");
618                 } else {
619                         rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
620                 
621                         printf("Tests: %d, Failures: %d", total, torture->results.failed);
622                         if (torture->results.failed - unexpected_failures) {
623                                 printf(" (%d expected)", torture->results.failed - unexpected_failures);
624                         }
625                         printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
626                            torture->results.errors, torture->results.skipped, rate);
627                 }
628
629                 if (unexpected_failures) {
630                         printf("The following tests failed:\n");
631                         for (i = 0; torture->results.unexpected_failures[i]; i++) {
632                                 printf("  %s\n", torture->results.unexpected_failures[i]);
633                         }
634                         printf("\n");
635                 }
636
637                 if (str_list_length(torture->results.unexpected_errors)) {
638                         printf("Errors occurred while running the following tests:\n");
639                         for (i = 0; torture->results.unexpected_errors[i]; i++) {
640                                 printf("  %s\n", torture->results.unexpected_errors[i]);
641                         }
642                         printf("\n");
643                 }
644
645                 if (str_list_length(torture->results.unexpected_successes)) {
646                         printf("The following tests were expected to fail but succeeded:\n");
647                         for (i = 0; torture->results.unexpected_successes[i]; i++) {
648                                 printf("  %s\n", torture->results.unexpected_successes[i]);
649                         }
650                         printf("\n");
651                 }
652         }
653
654         talloc_free(torture);
655
656         if (correct) {
657                 return(0);
658         } else {
659                 return(1);
660         }
661 }