nicer formatting
[tridge/junkcode.git] / tsm_torture.c
1 /*
2   test program to heavily stress a TSM/HSM system
3
4   Andrew Tridgell January 2008
5
6   compile with:
7
8      gcc -Wall -o tsm_torture{,.c} -lgpfs_gpl
9
10   If you want to use the -L or -S switches then you must symlink tsm_torture to smbd as 
11   otherwise it won't have permission to set share modes or leases
12
13      ln -s tsm_torture smbd
14
15   and run like this:
16
17     ./smbd /gpfs/data/tsmtest
18
19   where /gpfs/data/tsmtest is the directory to test on
20
21  */
22
23 #define _XOPEN_SOURCE 500
24 #define _GNU_SOURCE 
25
26 #include <stdio.h>
27 #include <unistd.h>
28 #include <stdlib.h>
29 #include <errno.h>
30 #include <sys/stat.h>
31 #include <fcntl.h>
32 #include <string.h>
33 #include <getopt.h>
34 #include <signal.h>
35 #include <utime.h>
36 #include <stdbool.h>
37 #include <sys/mman.h>
38 #include <sys/time.h>
39 #include <sys/wait.h>
40 #include <time.h>
41 #if WITH_GPFS
42 #include "gpfs_gpl.h"
43 #endif
44
45 static struct {
46         bool use_lease;
47         bool use_sharemode;
48         unsigned nprocesses;
49         unsigned nfiles;
50         unsigned timelimit;
51         unsigned fsize;
52         const char *dir;
53         const char *migrate_cmd;
54 } options = {
55         .use_lease     = false,
56         .use_sharemode = false,
57         .nprocesses    = 10,
58         .nfiles        = 10,
59         .fsize         = 8192,
60         .timelimit     = 30,
61         .migrate_cmd   = "dsmmigrate",
62 };
63
64 static pid_t parent_pid;
65
66 enum offline_op {OP_LOADFILE, OP_SAVEFILE, OP_MIGRATE, OP_GETOFFLINE, OP_ENDOFLIST};
67
68 struct child {
69         unsigned offline_count;
70         unsigned online_count;
71         unsigned migrate_fail_count;
72         unsigned io_fail_count;
73         unsigned migrate_ok_count;
74         unsigned count;
75         unsigned lastcount;
76         struct timeval tv_start;
77         double latencies[OP_ENDOFLIST];
78         double worst_latencies[OP_ENDOFLIST];
79         pid_t pid;
80         enum offline_op op;
81 };
82
83 static struct timeval tv_start;
84 static struct child *children;
85
86 static char *buf;
87
88 /* return a pointer to a /dev/zero shared memory segment of size "size"
89    which will persist across fork() but will disappear when all processes
90    exit 
91
92    The memory is zeroed automatically
93
94    This relies on /dev/zero being shared mmap capable, which it is
95    only under some OSes (Linux 2.1 _not_ included)
96  */
97 void *shm_setup(int size)
98 {
99         void *ret;
100         int fd;
101
102         fd = open("/dev/zero", O_RDWR);
103         if (fd == -1) {
104                 return NULL;
105         }
106         ret = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
107
108         /* note that we don't need to keep the file open */
109         close(fd);
110         if (ret == (void *)-1) return NULL;
111         return ret;
112 }
113
114
115 static struct timeval timeval_current(void)
116 {
117         struct timeval tv;
118         gettimeofday(&tv, NULL);
119         return tv;
120 }
121
122 static double timeval_elapsed(struct timeval *tv)
123 {
124         struct timeval tv2 = timeval_current();
125         return (tv2.tv_sec - tv->tv_sec) + 
126                (tv2.tv_usec - tv->tv_usec)*1.0e-6;
127 }
128
129 /*
130   file name given a number
131  */
132 static char *filename(int i)
133 {
134         char *s = NULL;
135         asprintf(&s, "%s/file%u.dat", options.dir, (unsigned)i);
136         return s;
137 }
138
139 static void sigio_handler(int sig)
140 {
141         printf("\nGot SIGIO\n");
142 }
143
144 /* 
145    load a file 
146  */
147 static void child_loadfile(struct child *child, const char *fname)
148 {
149         int fd;
150
151         signal(SIGIO, sigio_handler);
152
153         fd = open(fname, O_RDONLY);
154         if (fd == -1) {
155                 perror(fname);
156                 exit(1);
157         }
158
159 #if WITH_GPFS
160         if (options.use_lease && gpfs_set_lease(fd, GPFS_LEASE_READ) != 0) {
161                 printf("\ngpfs_set_lease on '%s' - %s\n", fname, strerror(errno));
162                 close(fd);
163                 return;
164         }
165         if (options.use_sharemode && gpfs_set_share(fd, 1, 2) != 0) {
166                 printf("\ngpfs_set_share on '%s' - %s\n", fname, strerror(errno));
167                 close(fd);
168                 return;
169         }
170 #endif
171
172         if (pread(fd, buf, options.fsize, 0) != options.fsize) {
173                 if (child->io_fail_count == 0) {
174                         printf("pread failed on '%s' - %s\n", fname, strerror(errno));
175                 }
176                 child->io_fail_count++;
177                 close(fd);
178                 return;
179         }
180
181         close(fd);
182 }
183
184
185 /* 
186    save a file 
187  */
188 static void child_savefile(struct child *child, const char *fname, unsigned fnumber)
189 {
190         int fd;
191
192         signal(SIGIO, sigio_handler);
193
194         fd = open(fname, O_WRONLY);
195         if (fd == -1) {
196                 perror(fname);
197                 exit(1);
198         }
199
200 #if WITH_GPFS
201         if (options.use_lease && gpfs_set_lease(fd, GPFS_LEASE_WRITE) != 0) {
202                 printf("\ngpfs_set_lease on '%s' - %s\n", fname, strerror(errno));
203                 close(fd);
204                 return;
205         }
206         if (options.use_sharemode && gpfs_set_share(fd, 1, 2) != 0) {
207                 printf("\ngpfs_set_share on '%s' - %s\n", fname, strerror(errno));
208                 close(fd);
209                 return;
210         }
211 #endif
212
213         memset(buf, fnumber%256, options.fsize);
214
215         if (pwrite(fd, buf, options.fsize, 0) != options.fsize) {
216                 if (child->io_fail_count == 0) {
217                         printf("pwrite failed on '%s' - %s\n", fname, strerror(errno));
218                 }
219                 child->io_fail_count++;
220                 close(fd);
221                 return;
222         }
223
224         fsync(fd);
225         close(fd);
226 }
227
228 /* 
229    get file offline status
230  */
231 static void child_getoffline(struct child *child, const char *fname)
232 {
233         struct stat st;
234         if (stat(fname, &st) != 0) {
235                 printf("\nFailed to stat '%s' - %s\n", fname, strerror(errno));
236                 exit(1);
237         }
238         if (st.st_size != options.fsize) {
239                 printf("\nWrong file size for '%s' - %u\n", fname, (unsigned)st.st_size);
240                 exit(1);
241         }
242         if (st.st_blocks == 0) {
243                 child->offline_count++;
244                 if (strcmp(options.migrate_cmd, "/bin/true") == 0) {
245                         printf("\nFile '%s' is offline with no migration command\n", fname);
246                 }
247         } else {
248                 child->online_count++;
249         }
250 }
251
252
253 /* 
254    set a file offline
255  */
256 static void child_migrate(struct child *child, const char *fname)
257 {
258         char *cmd = NULL;
259         int ret;
260         struct utimbuf t;
261         struct stat st;
262
263         if (stat(fname, &st) != 0) {
264                 printf("\nFailed to stat '%s' - %s\n", fname, strerror(errno));
265                 exit(1);
266         }
267         if (st.st_size != options.fsize) {
268                 printf("\nWrong file size for '%s' - %u\n", fname, (unsigned)st.st_size);
269                 exit(1);
270         }
271         if (st.st_blocks == 0) {
272                 /* already offline */
273                 return;
274         }
275
276         /* make the file a bit older so migation works */
277         t.actime = 0;
278         t.modtime = time(NULL) - 60*60;
279         utime(fname, &t);
280
281         asprintf(&cmd, "%s %s > /dev/null 2>&1", options.migrate_cmd, fname);
282         ret = system(cmd);
283         if (ret != -1) {
284                 ret = WEXITSTATUS(ret);
285         }
286         if (ret != 0) {
287                 children->migrate_fail_count++;
288         } else {
289                 children->migrate_ok_count++;
290         }
291         free(cmd);
292 }
293
294 /*
295   main child loop
296  */
297 static void run_child(struct child *child)
298 {
299         srandom(time(NULL) ^ getpid());
300
301         while (1) {
302                 double latency;
303                 unsigned fnumber = random() % options.nfiles;
304                 char *fname = filename(fnumber);
305
306                 if (kill(parent_pid, 0) != 0) {
307                         /* parent has exited */
308                         exit(0);
309                 }
310
311                 child->tv_start = timeval_current();
312
313                 child->op = random() % OP_ENDOFLIST;
314                 switch (child->op) {
315                 case OP_LOADFILE:
316                         child_loadfile(child, fname);
317                         break;
318                 case OP_SAVEFILE:
319                         child_savefile(child, fname, fnumber);
320                         break;
321                 case OP_MIGRATE:
322                         child_migrate(child, fname);
323                         break;
324                 case OP_GETOFFLINE:
325                         child_getoffline(child, fname);
326                         break;
327                 case OP_ENDOFLIST:
328                         break;
329                 }
330
331                 latency = timeval_elapsed(&child->tv_start);
332                 if (latency > child->latencies[child->op]) {
333                         child->latencies[child->op] = latency;
334                 }
335                 if (latency > child->worst_latencies[child->op]) {
336                         child->worst_latencies[child->op] = latency;
337                 }
338                 child->count++;
339
340                 free(fname);
341         }
342 }
343
344 static void sig_alarm(int sig)
345 {
346         int i, op;
347         unsigned total=0, total_offline=0, total_online=0, 
348                 total_migrate_failures=0, total_migrate_ok=0,
349                 total_io_failures=0;
350         double latencies[OP_ENDOFLIST];
351         double worst_latencies[OP_ENDOFLIST];
352         
353         if (timeval_elapsed(&tv_start) >= options.timelimit) {
354                 printf("\ntimelimit reached - killing children\n");
355                 for (i=0;i<options.nprocesses;i++) {
356                         kill(children[i].pid, SIGTERM);
357                 }
358         }
359
360         for (op=0;op<OP_ENDOFLIST;op++) {
361                 latencies[op] = 0;
362                 worst_latencies[op] = 0;
363         }
364
365         for (i=0;i<options.nprocesses;i++) {
366                 total += children[i].count - children[i].lastcount;
367                 children[i].lastcount = children[i].count;              
368                 total_online += children[i].online_count;
369                 total_offline += children[i].offline_count;
370                 total_migrate_failures += children[i].migrate_fail_count;
371                 total_io_failures += children[i].io_fail_count;
372                 total_migrate_ok += children[i].migrate_ok_count;
373                 for (op=0;op<OP_ENDOFLIST;op++) {
374                         if (children[i].latencies[op] > latencies[op]) {
375                                 latencies[op] = children[i].latencies[op];
376                         }
377                         children[i].latencies[op] = 0;
378                 }
379                 if (timeval_elapsed(&children[i].tv_start) > latencies[children[i].op]) {
380                         double lat;
381                         lat = timeval_elapsed(&children[i].tv_start);
382                         latencies[children[i].op] = lat;
383                         if (lat > worst_latencies[children[i].op]) {
384                                 worst_latencies[children[i].op] = lat;
385                         }
386                 }
387                 for (op=0;op<OP_ENDOFLIST;op++) {
388                         double lat = children[i].worst_latencies[op];
389                         if (lat > worst_latencies[op]) {
390                                 worst_latencies[op] = lat;
391                         }
392                 }
393         }
394
395         printf("ops/s=%4u offline=%u/%u  failures: mig=%u io=%u  latencies: mig=%.1f/%.1f stat=%.1f/%.1f write=%.1f/%.1f read=%.1f/%.1f                \r",
396                total, total_offline, total_online+total_offline, 
397                total_migrate_failures,
398                total_io_failures,
399                latencies[OP_MIGRATE], worst_latencies[OP_MIGRATE],
400                latencies[OP_GETOFFLINE], worst_latencies[OP_GETOFFLINE],
401                latencies[OP_SAVEFILE], worst_latencies[OP_SAVEFILE],
402                latencies[OP_LOADFILE], worst_latencies[OP_LOADFILE]);
403         fflush(stdout);
404         signal(SIGALRM, sig_alarm);
405         alarm(1);
406 }
407
408 static void usage(void)
409 {
410         printf("Usage: (note, must run as 'smbd' to use leases or share modes)\n");
411         printf("ln -sf tsm_torture smbd\n");
412         printf("./smbd [options] <directory>\n");
413         printf("Options:\n");
414         printf("  -N <nprocs>  number of child processes\n");
415         printf("  -F <nfiles>  number of files\n");
416         printf("  -t <time>    runtime (seconds)\n");
417         printf("  -s <fsize>   file size (bytes)\n");
418         printf("  -M <migrate> set file migrate command\n");
419         printf("  -L           use gpfs leases\n");
420         printf("  -S           use gpfs sharemodes\n");
421         exit(0);
422 }
423
424 int main(int argc, char * const argv[])
425 {
426         int opt, i;
427         const char *progname = argv[0];
428         struct stat st;
429
430         /* parse command-line options */
431         while ((opt = getopt(argc, argv, "LSN:F:t:s:M:h")) != -1) {
432                 switch (opt){
433                 case 'L':
434                         options.use_lease = true;
435                         break;
436                 case 'S':
437                         options.use_sharemode = true;
438                         break;
439                 case 'N':
440                         options.nprocesses = atoi(optarg);
441                         break;
442                 case 'F':
443                         options.nfiles = atoi(optarg);
444                         break;
445                 case 'M':
446                         options.migrate_cmd = strdup(optarg);
447                         break;
448                 case 's':
449                         options.fsize = atoi(optarg);
450                         break;
451                 case 't':
452                         options.timelimit = atoi(optarg);
453                         break;
454                 default:
455                         usage();
456                         break;
457                 }
458         }
459
460         if ((options.use_lease || options.use_sharemode) && strstr(progname, "smbd") == NULL) {
461                 printf("ERROR: you must invoke as smbd to use leases or share modes - use a symlink\n");
462                 exit(1);
463         }
464         
465
466         argv += optind;
467         argc -= optind;
468
469         if (argc == 0) {
470                 usage();
471         }
472
473         options.dir = argv[0];
474
475         if (stat(options.dir, &st) != 0 || !S_ISDIR(st.st_mode)) {
476                 printf("'%s' must exist and be a directory\n", options.dir);
477                 exit(1);
478         }
479
480         children = shm_setup(sizeof(*children) * options.nprocesses);
481
482         printf("Creating %u files of size %u in '%s'\n", 
483                options.nfiles, options.fsize, options.dir);
484
485         buf = malloc(options.fsize);
486
487         for (i=0;i<options.nfiles;i++) {
488                 int fd;
489                 char *fname = filename(i);
490                 fd = open(fname, O_CREAT|O_RDWR, 0600);
491                 if (fd == -1) {
492                         perror(fname);
493                         exit(1);
494                 }
495                 ftruncate(fd, options.fsize);
496                 memset(buf, i%256, options.fsize);
497                 if (write(fd, buf, options.fsize) != options.fsize) {
498                         printf("Failed to write '%s'\n", fname);
499                         exit(1);
500                 }
501                 fsync(fd);
502                 close(fd);
503                 free(fname);
504         }
505
506         parent_pid = getpid();
507
508         printf("Starting %u child processes for %u seconds\n", 
509                options.nprocesses, options.timelimit);
510         printf("Results shown as: offline=numoffline/total latencies: current/worst\n");
511
512         for (i=0;i<options.nprocesses;i++) {
513                 pid_t pid = fork();
514                 if (pid == 0) {
515                         children[i].pid = getpid();
516                         run_child(&children[i]);
517                 } else {
518                         children[i].pid = pid;
519                 }
520         }
521
522         /* show status once a second */
523         signal(SIGALRM, sig_alarm);
524         tv_start = timeval_current();
525         alarm(1);
526
527         /* wait for the children to finish */
528         for (i=0;i<options.nprocesses;i++) {
529                 while (waitpid(children[i].pid, 0, 0) != 0 && errno != ECHILD) ;
530         }       
531
532         return 0;
533 }