#include "system/time.h"
#include "system/wait.h"
#include "system/filesys.h"
+#include "system/readline.h"
+#include "lib/smbreadline/smbreadline.h"
#include "libcli/libcli.h"
#include "lib/ldb/include/ldb.h"
#include "lib/events/events.h"
+#include "dynconfig.h"
#include "torture/torture.h"
-#include "torture/ui.h"
#include "build.h"
-#include "dlinklist.h"
+#include "lib/util/dlinklist.h"
#include "librpc/rpc/dcerpc.h"
+static bool run_matching(struct torture_context *torture,
+ const char *prefix,
+ const char *expr,
+ struct torture_suite *suite,
+ bool *matched)
+{
+ bool ret = true;
+
+ if (suite == NULL) {
+ struct torture_suite *o;
+
+ for (o = torture_root->children; o; o = o->next) {
+ if (gen_fnmatch(expr, o->name) == 0) {
+ *matched = true;
+ init_iconv();
+ ret &= torture_run_suite(torture, o);
+ continue;
+ }
+
+ ret &= run_matching(torture, o->name, expr, o, matched);
+ }
+ } else {
+ char *name;
+ struct torture_suite *c;
+ struct torture_tcase *t;
+
+ for (c = suite->children; c; c = c->next) {
+ asprintf(&name, "%s-%s", prefix, c->name);
+
+ if (gen_fnmatch(expr, name) == 0) {
+ *matched = true;
+ init_iconv();
+ torture->active_testname = talloc_strdup(torture, prefix);
+ ret &= torture_run_suite(torture, c);
+ free(name);
+ continue;
+ }
+
+ ret &= run_matching(torture, name, expr, c, matched);
+
+ free(name);
+ }
+
+ for (t = suite->testcases; t; t = t->next) {
+ asprintf(&name, "%s-%s", prefix, t->name);
+ if (gen_fnmatch(expr, name) == 0) {
+ *matched = true;
+ init_iconv();
+ torture->active_testname = talloc_strdup(torture, prefix);
+ ret &= torture_run_tcase(torture, t);
+ talloc_free(torture->active_testname);
+ }
+ free(name);
+ }
+ }
+
+ return ret;
+}
+
#define MAX_COLS 80 /* FIXME: Determine this at run-time */
/****************************************************************************
run a specified test or "ALL"
****************************************************************************/
-static BOOL run_test(struct torture_context *torture, const char *name)
+static bool run_test(struct torture_context *torture, const char *name)
{
- BOOL ret = True;
- struct torture_op *o;
- BOOL matched = False;
-
- if (strequal(name,"ALL")) {
- for (o = torture_ops; o; o = o->next) {
- if (!run_test(torture, o->name)) {
- ret = False;
- }
+ bool ret = true;
+ bool matched = false;
+ struct torture_suite *o;
+
+ if (strequal(name, "ALL")) {
+ for (o = torture_root->children; o; o = o->next) {
+ ret &= torture_run_suite(torture, o);
}
return ret;
}
- for (o = torture_ops; o; o = o->next) {
- if (gen_fnmatch(name, o->name) == 0) {
- double t;
- matched = True;
- init_iconv();
- printf("Running %s\n", o->name);
- if (o->multi_fn) {
- BOOL result = False;
- t = torture_create_procs(o->multi_fn,
- &result);
- if (!result) {
- ret = False;
- printf("TEST %s FAILED!\n", o->name);
- }
-
- } else {
- struct timeval tv = timeval_current();
- if (!o->fn(torture)) {
- ret = False;
- printf("TEST %s FAILED!\n", o->name);
- }
- t = timeval_elapsed(&tv);
- }
- printf("%s took %g secs\n\n", o->name, t);
- }
- }
+ ret = run_matching(torture, NULL, name, NULL, &matched);
if (!matched) {
printf("Unknown torture operation '%s'\n", name);
- ret = False;
+ return false;
}
return ret;
static void usage(poptContext pc)
{
- struct torture_op *o;
+ struct torture_suite *o;
+ struct torture_suite *s;
+ struct torture_tcase *t;
int i;
poptPrintUsage(pc, stdout, 0);
printf(" TRANSPORT:host[flags]\n\n");
- printf(" where TRANSPORT is either ncacn_np for SMB or ncacn_ip_tcp for RPC/TCP\n\n");
+ printf(" where TRANSPORT is either ncacn_np for SMB, ncacn_ip_tcp for RPC/TCP\n");
+ printf(" or ncalrpc for local connections.\n\n");
printf(" 'host' is an IP or hostname or netbios name. If the binding string\n");
printf(" identifies the server side of an endpoint, 'host' may be an empty\n");
printf(" ncacn_ip_tcp:myserver[1024]\n");
printf(" ncacn_ip_tcp:myserver[1024,sign,seal]\n\n");
- printf("The unc format is:\n\n");
+ printf(" ncalrpc:\n\n");
- printf(" //server/share\n\n");
+ printf("The UNC format is:\n\n");
- printf("tests are:\n");
+ printf(" //server/share\n\n");
- i = 0;
- for (o = torture_ops; o; o = o->next) {
- if (i + strlen(o->name) >= MAX_COLS) {
- printf("\n");
- i = 0;
+ printf("Tests are:");
+
+ for (o = torture_root->children; o; o = o->next) {
+ printf("\n%s (%s):\n ", o->description, o->name);
+
+ i = 0;
+ for (s = o->children; s; s = s->next) {
+ if (i + strlen(o->name) + strlen(s->name) >= (MAX_COLS - 3)) {
+ printf("\n ");
+ i = 0;
+ }
+ i+=printf("%s-%s ", o->name, s->name);
}
- i+=printf("%s ", o->name);
- }
- printf("\n\n");
- printf("default test is ALL\n");
+ for (t = o->testcases; t; t = t->next) {
+ if (i + strlen(o->name) + strlen(t->name) >= (MAX_COLS - 3)) {
+ printf("\n ");
+ i = 0;
+ }
+ i+=printf("%s-%s ", o->name, t->name);
+ }
- exit(1);
-}
+ if (i) printf("\n");
+ }
-static BOOL is_binding_string(const char *binding_string)
-{
- TALLOC_CTX *mem_ctx = talloc_init("is_binding_string");
- struct dcerpc_binding *binding_struct;
- NTSTATUS status;
-
- status = dcerpc_parse_binding(mem_ctx, binding_string, &binding_struct);
+ printf("\nThe default test is ALL.\n");
- talloc_free(mem_ctx);
- return NT_STATUS_IS_OK(status);
+ exit(1);
}
static void max_runtime_handler(int sig)
exit(1);
}
-static void simple_test_start (struct torture_test *test)
+struct timeval last_suite_started;
+
+static void simple_suite_start(struct torture_context *ctx,
+ struct torture_suite *suite)
{
- printf("Testing %s...\n", test->name);
+ last_suite_started = timeval_current();
+ printf("Running %s\n", suite->name);
}
-static void simple_test_result (struct torture_test *test, enum torture_result res)
+static void simple_suite_finish(struct torture_context *ctx,
+ struct torture_suite *suite)
{
- printf("\t %d\n",res);
+
+ printf("%s took %g secs\n\n", suite->name,
+ timeval_elapsed(&last_suite_started));
}
-static void simple_comment (struct torture_test *test, const char *comment)
+static void simple_test_result (struct torture_context *context,
+ enum torture_result res, const char *reason)
{
- printf("# %s\n", comment);
+ switch (res) {
+ case TORTURE_OK:
+ if (reason)
+ printf("OK: %s\n", reason);
+ break;
+ case TORTURE_FAIL:
+ printf("TEST %s FAILED! - %s\n", context->active_test->name, reason);
+ break;
+ case TORTURE_ERROR:
+ printf("ERROR IN TEST %s! - %s\n", context->active_test->name, reason);
+ break;
+ case TORTURE_SKIP:
+ printf("SKIP: %s - %s\n", context->active_test->name, reason);
+ break;
+ }
+}
+
+static void simple_comment (struct torture_context *test,
+ const char *comment)
+{
+ printf("%s", comment);
}
const static struct torture_ui_ops std_ui_ops = {
.comment = simple_comment,
- .test_start = simple_test_start,
+ .suite_start = simple_suite_start,
+ .suite_finish = simple_suite_finish,
.test_result = simple_test_result
};
+static void subunit_suite_start(struct torture_context *ctx,
+ struct torture_suite *suite)
+{
+ printf("testsuite: %s\n", suite->path);
+}
+
+static void subunit_test_start (struct torture_context *ctx,
+ struct torture_tcase *tcase,
+ struct torture_test *test)
+{
+ printf("test: %s\n", test->name);
+}
+
+static void subunit_test_result (struct torture_context *context,
+ enum torture_result res, const char *reason)
+{
+ switch (res) {
+ case TORTURE_OK:
+ printf("success: %s", context->active_test->name);
+ break;
+ case TORTURE_FAIL:
+ printf("failure: %s", context->active_test->name);
+ break;
+ case TORTURE_ERROR:
+ printf("error: %s", context->active_test->name);
+ break;
+ case TORTURE_SKIP:
+ printf("skip: %s", context->active_test->name);
+ break;
+ }
+ if (reason)
+ printf(" [ %s ]", reason);
+ printf("\n");
+}
+
+static void subunit_comment (struct torture_context *test,
+ const char *comment)
+{
+ fprintf(stderr, "%s", comment);
+}
+
+const static struct torture_ui_ops subunit_ui_ops = {
+ .comment = subunit_comment,
+ .test_start = subunit_test_start,
+ .test_result = subunit_test_result,
+ .suite_start = subunit_suite_start
+};
+
+static void harness_test_start (struct torture_context *ctx,
+ struct torture_tcase *tcase,
+ struct torture_test *test)
+{
+}
+
+static void harness_test_result (struct torture_context *context,
+ enum torture_result res, const char *reason)
+{
+ switch (res) {
+ case TORTURE_OK:
+ printf("ok %s - %s\n", context->active_test->name, reason);
+ break;
+ case TORTURE_FAIL:
+ case TORTURE_ERROR:
+ printf("not ok %s - %s\n", context->active_test->name, reason);
+ break;
+ case TORTURE_SKIP:
+ printf("skip %s - %s\n", context->active_test->name, reason);
+ break;
+ }
+}
+
+static void harness_comment (struct torture_context *test,
+ const char *comment)
+{
+ printf("# %s\n", comment);
+}
+
+const static struct torture_ui_ops harness_ui_ops = {
+ .comment = harness_comment,
+ .test_start = harness_test_start,
+ .test_result = harness_test_result
+};
+
+static void quiet_suite_start(struct torture_context *ctx,
+ struct torture_suite *suite)
+{
+ int i;
+ ctx->quiet = true;
+ for (i = 1; i < ctx->level; i++) putchar('\t');
+ printf("%s: ", suite->name);
+ fflush(stdout);
+}
+
+static void quiet_suite_finish(struct torture_context *ctx,
+ struct torture_suite *suite)
+{
+ putchar('\n');
+}
+
+static void quiet_test_result (struct torture_context *context,
+ enum torture_result res, const char *reason)
+{
+ fflush(stdout);
+ switch (res) {
+ case TORTURE_OK: putchar('.'); break;
+ case TORTURE_FAIL: putchar('F'); break;
+ case TORTURE_ERROR: putchar('E'); break;
+ case TORTURE_SKIP: putchar('I'); break;
+ }
+}
+
+const static struct torture_ui_ops quiet_ui_ops = {
+ .suite_start = quiet_suite_start,
+ .suite_finish = quiet_suite_finish,
+ .test_result = quiet_test_result
+};
+
+void run_recipe(struct torture_context *tctx, const char *recipe)
+{
+ int numlines, i, ret;
+ char **lines;
+
+ lines = file_lines_load(recipe, &numlines, NULL);
+ if (lines == NULL) {
+ fprintf(stderr, "Unable to load file %s\n", recipe);
+ return;
+ }
+
+ for (i = 0; i < numlines; i++) {
+ int argc;
+ const char **argv;
+
+ ret = poptParseArgvString(lines[i], &argc, &argv);
+ if (ret != 0) {
+ fprintf(stderr, "Error parsing line\n");
+ continue;
+ }
+
+ run_test(tctx, argv[0]);
+ }
+
+ talloc_free(lines);
+}
+
+void run_shell(struct torture_context *tctx)
+{
+ char *cline;
+ int argc;
+ const char **argv;
+ int ret;
+
+ while (1) {
+ cline = smb_readline("torture> ", NULL, NULL);
+
+ if (cline == NULL)
+ return;
+
+ ret = poptParseArgvString(cline, &argc, &argv);
+ if (ret != 0) {
+ fprintf(stderr, "Error parsing line\n");
+ continue;
+ }
+
+ if (!strcmp(argv[0], "quit")) {
+ return;
+ } else if (!strcmp(argv[0], "set")) {
+ if (argc < 3) {
+ fprintf(stderr, "Usage: set <variable> <value>\n");
+ } else {
+ char *name = talloc_asprintf(NULL, "torture:%s", argv[1]);
+ lp_set_cmdline(name, argv[2]);
+ talloc_free(name);
+ }
+ } else if (!strcmp(argv[0], "help")) {
+ fprintf(stderr, "Available commands:\n"
+ " help - This help command\n"
+ " run - Run test\n"
+ " set - Change variables\n"
+ "\n");
+ } else if (!strcmp(argv[0], "run")) {
+ if (argc < 2) {
+ fprintf(stderr, "Usage: run TEST-NAME [OPTIONS...]\n");
+ } else {
+ run_test(tctx, argv[1]);
+ }
+ }
+ }
+}
+
/****************************************************************************
main program
****************************************************************************/
- int main(int argc,char *argv[])
+int main(int argc,char *argv[])
{
int opt, i;
- char *p;
- BOOL correct = True;
+ bool correct = true;
int max_runtime=0;
int argc_new;
struct torture_context *torture;
+ const struct torture_ui_ops *ui_ops;
char **argv_new;
poptContext pc;
+ static const char *target = "other";
+ const char **subunit_dir;
+ struct dcerpc_binding *binding_struct;
+ NTSTATUS status;
+ int shell = False;
+ static const char *ui_ops_name = "simple";
enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
- OPT_DANGEROUS,OPT_SMB_PORTS};
+ OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC,OPT_NUMPROGS, OPT_BASEDIR};
struct poptOption long_options[] = {
POPT_AUTOHELP
+ {"format", 0, POPT_ARG_STRING, &ui_ops_name, 0, "Output format (one of: simple, subunit, harness)", NULL },
{"smb-ports", 'p', POPT_ARG_STRING, NULL, OPT_SMB_PORTS, "SMB ports", NULL},
+ {"basedir", 0, POPT_ARG_STRING, NULL, OPT_BASEDIR, "base directory", "BSAEDIR" },
{"seed", 0, POPT_ARG_INT, &torture_seed, 0, "seed", NULL},
- {"num-progs", 0, POPT_ARG_INT, &torture_nprocs, 0, "num progs", NULL},
+ {"num-progs", 0, POPT_ARG_INT, NULL, OPT_NUMPROGS, "num progs", NULL},
{"num-ops", 0, POPT_ARG_INT, &torture_numops, 0, "num ops", NULL},
{"entries", 0, POPT_ARG_INT, &torture_entries, 0, "entries", NULL},
- {"use-oplocks", 'L', POPT_ARG_NONE, &use_oplocks, 0, "use oplocks", NULL},
- {"show-all", 0, POPT_ARG_NONE, &torture_showall, 0, "show all", NULL},
{"loadfile", 0, POPT_ARG_STRING, NULL, OPT_LOADFILE, "loadfile", NULL},
{"unclist", 0, POPT_ARG_STRING, NULL, OPT_UNCLIST, "unclist", NULL},
- {"timelimit", 't', POPT_ARG_STRING, NULL, OPT_TIMELIMIT, "timelimit", NULL},
+ {"timelimit", 't', POPT_ARG_INT, NULL, OPT_TIMELIMIT, "timelimit", NULL},
{"failures", 'f', POPT_ARG_INT, &torture_failures, 0, "failures", NULL},
{"parse-dns", 'D', POPT_ARG_STRING, NULL, OPT_DNS, "parse-dns", NULL},
- {"dangerous", 'X', POPT_ARG_NONE, NULL, OPT_DANGEROUS, "dangerous", NULL},
+ {"dangerous", 'X', POPT_ARG_NONE, NULL, OPT_DANGEROUS,
+ "run dangerous tests (eg. wiping out password database)", NULL},
+ {"shell", 0, POPT_ARG_NONE, &shell, True, "Run shell", NULL},
+ {"target", 'T', POPT_ARG_STRING, &target, 0, "samba3|samba4|other", NULL},
+ {"async", 'a', POPT_ARG_NONE, NULL, OPT_ASYNC,
+ "run async tests", NULL},
+ {"num-async", 0, POPT_ARG_INT, &torture_numasync, 0,
+ "number of simultaneous async requests", NULL},
{"maximum-runtime", 0, POPT_ARG_INT, &max_runtime, 0,
"set maximum time for smbtorture to live", "seconds"},
POPT_COMMON_SAMBA
POPT_COMMON_CONNECTION
POPT_COMMON_CREDENTIALS
POPT_COMMON_VERSION
- POPT_TABLEEND
+ { NULL }
};
-#ifdef HAVE_SETBUFFER
- setbuffer(stdout, NULL, 0);
-#endif
-
- torture_init();
+ setlinebuf(stdout);
/* we are never interested in SIGPIPE */
- BlockSignals(True,SIGPIPE);
+ BlockSignals(true, SIGPIPE);
pc = poptGetContext("smbtorture", argc, (const char **) argv, long_options,
POPT_CONTEXT_KEEP_FIRST);
case OPT_TIMELIMIT:
lp_set_cmdline("torture:timelimit", poptGetOptArg(pc));
break;
+ case OPT_NUMPROGS:
+ lp_set_cmdline("torture:nprocs", poptGetOptArg(pc));
+ break;
+ case OPT_BASEDIR:
+ lp_set_cmdline("torture:basedir", poptGetOptArg(pc));
+ break;
case OPT_DNS:
parse_dns(poptGetOptArg(pc));
break;
case OPT_DANGEROUS:
lp_set_cmdline("torture:dangerous", "Yes");
break;
+ case OPT_ASYNC:
+ lp_set_cmdline("torture:async", "Yes");
+ break;
case OPT_SMB_PORTS:
lp_set_cmdline("smb ports", poptGetOptArg(pc));
break;
- default:
- d_printf("Invalid option %s: %s\n",
- poptBadOption(pc, 0), poptStrerror(opt));
- usage(pc);
- exit(1);
}
}
+ if (strcmp(target, "samba3") == 0) {
+ lp_set_cmdline("torture:samba3", "true");
+ lp_set_cmdline("torture:knownfail", "samba3-knownfail");
+ } else if (strcmp(target, "samba4") == 0) {
+ lp_set_cmdline("torture:samba4", "true");
+ lp_set_cmdline("torture:knownfail", "samba4-knownfail");
+ }
+
if (max_runtime) {
/* this will only work if nobody else uses alarm(),
which means it won't work for some tests, but we
alarm(max_runtime);
}
+ torture_init();
ldb_global_init();
+ subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
+ if (subunit_dir == NULL)
+ torture_subunit_load_testsuites(dyn_TORTUREDIR, true, NULL);
+ else {
+ for (i = 0; subunit_dir[i]; i++)
+ torture_subunit_load_testsuites(subunit_dir[i], true, NULL);
+ }
+
if (torture_seed == 0) {
torture_seed = time(NULL);
}
}
}
- if (argc_new < 3) {
+ if (!(argc_new >= 3 || (shell && argc_new >= 2))) {
usage(pc);
exit(1);
}
- for(p = argv_new[1]; *p; p++) {
- if(*p == '\\')
- *p = '/';
- }
-
/* see if its a RPC transport specifier */
- if (is_binding_string(argv_new[1])) {
+ status = dcerpc_parse_binding(talloc_autofree_context(), argv_new[1], &binding_struct);
+ if (NT_STATUS_IS_OK(status)) {
+ lp_set_cmdline("torture:host", binding_struct->host);
+ lp_set_cmdline("torture:share", "IPC$");
lp_set_cmdline("torture:binding", argv_new[1]);
} else {
char *binding = NULL;
lp_set_cmdline("torture:binding", binding);
}
- torture = talloc_zero(NULL, struct torture_context);
- torture->ui_ops = &std_ui_ops;
+ if (!strcmp(ui_ops_name, "simple")) {
+ ui_ops = &std_ui_ops;
+ } else if (!strcmp(ui_ops_name, "subunit")) {
+ ui_ops = &subunit_ui_ops;
+ } else if (!strcmp(ui_ops_name, "harness")) {
+ ui_ops = &harness_ui_ops;
+ } else if (!strcmp(ui_ops_name, "quiet")) {
+ ui_ops = &quiet_ui_ops;
+ } else {
+ printf("Unknown output format '%s'\n", ui_ops_name);
+ exit(1);
+ }
+
+ torture = torture_context_init(talloc_autofree_context(),
+ lp_parm_string(-1, "torture", "knownfail"), ui_ops);
if (argc_new == 0) {
printf("You must specify a test to run, or 'ALL'\n");
+ } else if (shell) {
+ run_shell(torture);
} else {
+ int total;
+ double rate;
+ int unexpected_failures;
for (i=2;i<argc_new;i++) {
- if (!run_test(torture, argv_new[i])) {
- correct = False;
+ if (argv_new[i][0] == '@') {
+ run_recipe(torture, argv_new[i]+1);
+ } else if (!run_test(torture, argv_new[i])) {
+ correct = false;
}
}
- }
- talloc_free(torture);
+ unexpected_failures = str_list_length(torture->results.unexpected_failures);
+
+ total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
+ if (total == 0) {
+ printf("No tests run.\n");
+ } else {
+ rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
+
+ printf("Tests: %d, Failures: %d", total, torture->results.failed);
+ if (torture->results.failed - unexpected_failures) {
+ printf(" (%d expected)", torture->results.failed - unexpected_failures);
+ }
+ printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
+ torture->results.errors, torture->results.skipped, rate);
+ }
+
+ if (unexpected_failures) {
+ printf("The following tests failed:\n");
+ for (i = 0; torture->results.unexpected_failures[i]; i++) {
+ printf(" %s\n", torture->results.unexpected_failures[i]);
+ }
+ printf("\n");
+ }
+
+ if (str_list_length(torture->results.unexpected_errors)) {
+ printf("Errors occurred while running the following tests:\n");
+ for (i = 0; torture->results.unexpected_errors[i]; i++) {
+ printf(" %s\n", torture->results.unexpected_errors[i]);
+ }
+ printf("\n");
+ }
+
+ if (str_list_length(torture->results.unexpected_successes)) {
+ printf("The following tests were expected to fail but succeeded:\n");
+ for (i = 0; torture->results.unexpected_successes[i]; i++) {
+ printf(" %s\n", torture->results.unexpected_successes[i]);
+ }
+ printf("\n");
+ }
+ }
- if (correct) {
+ if (torture->results.returncode) {
return(0);
} else {
return(1);