drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13 #include <linux/delay.h>
  14 #include <linux/dma-mapping.h>
  15 #include <linux/dmaengine.h>
  16 #include <linux/freezer.h>
  17 #include <linux/init.h>
  18 #include <linux/kthread.h>
  19 #include <linux/sched/task.h>
  20 #include <linux/module.h>
  21 #include <linux/moduleparam.h>
  22 #include <linux/random.h>
  23 #include <linux/slab.h>
  24 #include <linux/wait.h>
  25
  26 static unsigned int test_buf_size = 16384;
  27 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  28 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  29
  30 static char test_device[32];
  31 module_param_string(device, test_device, sizeof(test_device),
  32                 S_IRUGO | S_IWUSR);
  33 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  34
  35 static unsigned int threads_per_chan = 1;
  36 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  37 MODULE_PARM_DESC(threads_per_chan,
  38                 "Number of threads to start per channel (default: 1)");
  39
  40 static unsigned int max_channels;
  41 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  42 MODULE_PARM_DESC(max_channels,
  43                 "Maximum number of channels to use (default: all)");
  44
  45 static unsigned int iterations;
  46 module_param(iterations, uint, S_IRUGO | S_IWUSR);
  47 MODULE_PARM_DESC(iterations,
  48                 "Iterations before stopping test (default: infinite)");
  49
  50 static unsigned int dmatest;
  51 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
  52 MODULE_PARM_DESC(dmatest,
  53                 "dmatest 0-memcpy 1-memset (default: 0)");
  54
  55 static unsigned int xor_sources = 3;
  56 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  57 MODULE_PARM_DESC(xor_sources,
  58                 "Number of xor source buffers (default: 3)");
  59
  60 static unsigned int pq_sources = 3;
  61 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  62 MODULE_PARM_DESC(pq_sources,
  63                 "Number of p+q source buffers (default: 3)");
  64
  65 static int timeout = 3000;
  66 module_param(timeout, uint, S_IRUGO | S_IWUSR);
  67 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  68                  "Pass -1 for infinite timeout");
  69
  70 static bool noverify;
  71 module_param(noverify, bool, S_IRUGO | S_IWUSR);
  72 MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
  73
  74 static bool norandom;
  75 module_param(norandom, bool, 0644);
  76 MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
  77
  78 static bool verbose;
  79 module_param(verbose, bool, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
  81
  82 static int alignment = -1;
  83 module_param(alignment, int, 0644);
  84 MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
  85
  86 static unsigned int transfer_size;
  87 module_param(transfer_size, uint, 0644);
  88 MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
  89
  90 /**
  91  * struct dmatest_params - test parameters.
  92  * @buf_size:           size of the memcpy test buffer
  93  * @channel:            bus ID of the channel to test
  94  * @device:             bus ID of the DMA Engine to test
  95  * @threads_per_chan:   number of threads to start per channel
  96  * @max_channels:       maximum number of channels to use
  97  * @iterations:         iterations before stopping test
  98  * @xor_sources:        number of xor source buffers
  99  * @pq_sources:         number of p+q source buffers
 100  * @timeout:            transfer timeout in msec, -1 for infinite timeout
 101  */
 102 struct dmatest_params {
 103         unsigned int    buf_size;
 104         char            channel[20];
 105         char            device[32];
 106         unsigned int    threads_per_chan;
 107         unsigned int    max_channels;
 108         unsigned int    iterations;
 109         unsigned int    xor_sources;
 110         unsigned int    pq_sources;
 111         int             timeout;
 112         bool            noverify;
 113         bool            norandom;
 114         int             alignment;
 115         unsigned int    transfer_size;
 116 };
 117
 118 /**
 119  * struct dmatest_info - test information.
 120  * @params:             test parameters
 121  * @lock:               access protection to the fields of this structure
 122  */
 123 static struct dmatest_info {
 124         /* Test parameters */
 125         struct dmatest_params   params;
 126
 127         /* Internal state */
 128         struct list_head        channels;
 129         unsigned int            nr_channels;
 130         struct mutex            lock;
 131         bool                    did_init;
 132 } test_info = {
 133         .channels = LIST_HEAD_INIT(test_info.channels),
 134         .lock = __MUTEX_INITIALIZER(test_info.lock),
 135 };
 136
 137 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 138 static int dmatest_run_get(char *val, const struct kernel_param *kp);
 139 static const struct kernel_param_ops run_ops = {
 140         .set = dmatest_run_set,
 141         .get = dmatest_run_get,
 142 };
 143 static bool dmatest_run;
 144 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
 145 MODULE_PARM_DESC(run, "Run the test (default: false)");
 146
 147 static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
 148 static int dmatest_chan_get(char *val, const struct kernel_param *kp);
 149 static const struct kernel_param_ops multi_chan_ops = {
 150         .set = dmatest_chan_set,
 151         .get = dmatest_chan_get,
 152 };
 153
 154 static char test_channel[20];
 155 static struct kparam_string newchan_kps = {
 156         .string = test_channel,
 157         .maxlen = 20,
 158 };
 159 module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
 160 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
 161
 162 static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
 163 static const struct kernel_param_ops test_list_ops = {
 164         .get = dmatest_test_list_get,
 165 };
 166 module_param_cb(test_list, &test_list_ops, NULL, 0444);
 167 MODULE_PARM_DESC(test_list, "Print current test list");
 168
 169 /* Maximum amount of mismatched bytes in buffer to print */
 170 #define MAX_ERROR_COUNT         32
 171
 172 /*
 173  * Initialization patterns. All bytes in the source buffer has bit 7
 174  * set, all bytes in the destination buffer has bit 7 cleared.
 175  *
 176  * Bit 6 is set for all bytes which are to be copied by the DMA
 177  * engine. Bit 5 is set for all bytes which are to be overwritten by
 178  * the DMA engine.
 179  *
 180  * The remaining bits are the inverse of a counter which increments by
 181  * one for each byte address.
 182  */
 183 #define PATTERN_SRC             0x80
 184 #define PATTERN_DST             0x00
 185 #define PATTERN_COPY            0x40
 186 #define PATTERN_OVERWRITE       0x20
 187 #define PATTERN_COUNT_MASK      0x1f
 188 #define PATTERN_MEMSET_IDX      0x01
 189
 190 /* Fixed point arithmetic ops */
 191 #define FIXPT_SHIFT             8
 192 #define FIXPNT_MASK             0xFF
 193 #define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
 194 #define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
 195 #define FIXPT_GET_FRAC(a)       ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
 196
 197 /* poor man's completion - we want to use wait_event_freezable() on it */
 198 struct dmatest_done {
 199         bool                    done;
 200         wait_queue_head_t       *wait;
 201 };
 202
 203 struct dmatest_data {
 204         u8              **raw;
 205         u8              **aligned;
 206         unsigned int    cnt;
 207         unsigned int    off;
 208 };
 209
 210 struct dmatest_thread {
 211         struct list_head        node;
 212         struct dmatest_info     *info;
 213         struct task_struct      *task;
 214         struct dma_chan         *chan;
 215         struct dmatest_data     src;
 216         struct dmatest_data     dst;
 217         enum dma_transaction_type type;
 218         wait_queue_head_t done_wait;
 219         struct dmatest_done test_done;
 220         bool                    done;
 221         bool                    pending;
 222 };
 223
 224 struct dmatest_chan {
 225         struct list_head        node;
 226         struct dma_chan         *chan;
 227         struct list_head        threads;
 228 };
 229
 230 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
 231 static bool wait;
 232
 233 static bool is_threaded_test_run(struct dmatest_info *info)
 234 {
 235         struct dmatest_chan *dtc;
 236
 237         list_for_each_entry(dtc, &info->channels, node) {
 238                 struct dmatest_thread *thread;
 239
 240                 list_for_each_entry(thread, &dtc->threads, node) {
 241                         if (!thread->done)
 242                                 return true;
 243                 }
 244         }
 245
 246         return false;
 247 }
 248
 249 static bool is_threaded_test_pending(struct dmatest_info *info)
 250 {
 251         struct dmatest_chan *dtc;
 252
 253         list_for_each_entry(dtc, &info->channels, node) {
 254                 struct dmatest_thread *thread;
 255
 256                 list_for_each_entry(thread, &dtc->threads, node) {
 257                         if (thread->pending)
 258                                 return true;
 259                 }
 260         }
 261
 262         return false;
 263 }
 264
 265 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
 266 {
 267         struct dmatest_info *info = &test_info;
 268         struct dmatest_params *params = &info->params;
 269
 270         if (params->iterations)
 271                 wait_event(thread_wait, !is_threaded_test_run(info));
 272         wait = true;
 273         return param_get_bool(val, kp);
 274 }
 275
 276 static const struct kernel_param_ops wait_ops = {
 277         .get = dmatest_wait_get,
 278         .set = param_set_bool,
 279 };
 280 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
 281 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
 282
 283 static bool dmatest_match_channel(struct dmatest_params *params,
 284                 struct dma_chan *chan)
 285 {
 286         if (params->channel[0] == '\0')
 287                 return true;
 288         return strcmp(dma_chan_name(chan), params->channel) == 0;
 289 }
 290
 291 static bool dmatest_match_device(struct dmatest_params *params,
 292                 struct dma_device *device)
 293 {
 294         if (params->device[0] == '\0')
 295                 return true;
 296         return strcmp(dev_name(device->dev), params->device) == 0;
 297 }
 298
 299 static unsigned long dmatest_random(void)
 300 {
 301         unsigned long buf;
 302
 303         prandom_bytes(&buf, sizeof(buf));
 304         return buf;
 305 }
 306
 307 static inline u8 gen_inv_idx(u8 index, bool is_memset)
 308 {
 309         u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
 310
 311         return ~val & PATTERN_COUNT_MASK;
 312 }
 313
 314 static inline u8 gen_src_value(u8 index, bool is_memset)
 315 {
 316         return PATTERN_SRC | gen_inv_idx(index, is_memset);
 317 }
 318
 319 static inline u8 gen_dst_value(u8 index, bool is_memset)
 320 {
 321         return PATTERN_DST | gen_inv_idx(index, is_memset);
 322 }
 323
 324 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 325                 unsigned int buf_size, bool is_memset)
 326 {
 327         unsigned int i;
 328         u8 *buf;
 329
 330         for (; (buf = *bufs); bufs++) {
 331                 for (i = 0; i < start; i++)
 332                         buf[i] = gen_src_value(i, is_memset);
 333                 for ( ; i < start + len; i++)
 334                         buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
 335                 for ( ; i < buf_size; i++)
 336                         buf[i] = gen_src_value(i, is_memset);
 337                 buf++;
 338         }
 339 }
 340
 341 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 342                 unsigned int buf_size, bool is_memset)
 343 {
 344         unsigned int i;
 345         u8 *buf;
 346
 347         for (; (buf = *bufs); bufs++) {
 348                 for (i = 0; i < start; i++)
 349                         buf[i] = gen_dst_value(i, is_memset);
 350                 for ( ; i < start + len; i++)
 351                         buf[i] = gen_dst_value(i, is_memset) |
 352                                                 PATTERN_OVERWRITE;
 353                 for ( ; i < buf_size; i++)
 354                         buf[i] = gen_dst_value(i, is_memset);
 355         }
 356 }
 357
 358 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 359                 unsigned int counter, bool is_srcbuf, bool is_memset)
 360 {
 361         u8              diff = actual ^ pattern;
 362         u8              expected = pattern | gen_inv_idx(counter, is_memset);
 363         const char      *thread_name = current->comm;
 364
 365         if (is_srcbuf)
 366                 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 367                         thread_name, index, expected, actual);
 368         else if ((pattern & PATTERN_COPY)
 369                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 370                 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 371                         thread_name, index, expected, actual);
 372         else if (diff & PATTERN_SRC)
 373                 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 374                         thread_name, index, expected, actual);
 375         else
 376                 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 377                         thread_name, index, expected, actual);
 378 }
 379
 380 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 381                 unsigned int end, unsigned int counter, u8 pattern,
 382                 bool is_srcbuf, bool is_memset)
 383 {
 384         unsigned int i;
 385         unsigned int error_count = 0;
 386         u8 actual;
 387         u8 expected;
 388         u8 *buf;
 389         unsigned int counter_orig = counter;
 390
 391         for (; (buf = *bufs); bufs++) {
 392                 counter = counter_orig;
 393                 for (i = start; i < end; i++) {
 394                         actual = buf[i];
 395                         expected = pattern | gen_inv_idx(counter, is_memset);
 396                         if (actual != expected) {
 397                                 if (error_count < MAX_ERROR_COUNT)
 398                                         dmatest_mismatch(actual, pattern, i,
 399                                                          counter, is_srcbuf,
 400                                                          is_memset);
 401                                 error_count++;
 402                         }
 403                         counter++;
 404                 }
 405         }
 406
 407         if (error_count > MAX_ERROR_COUNT)
 408                 pr_warn("%s: %u errors suppressed\n",
 409                         current->comm, error_count - MAX_ERROR_COUNT);
 410
 411         return error_count;
 412 }
 413
 414
 415 static void dmatest_callback(void *arg)
 416 {
 417         struct dmatest_done *done = arg;
 418         struct dmatest_thread *thread =
 419                 container_of(done, struct dmatest_thread, test_done);
 420         if (!thread->done) {
 421                 done->done = true;
 422                 wake_up_all(done->wait);
 423         } else {
 424                 /*
 425                  * If thread->done, it means that this callback occurred
 426                  * after the parent thread has cleaned up. This can
 427                  * happen in the case that driver doesn't implement
 428                  * the terminate_all() functionality and a dma operation
 429                  * did not occur within the timeout period
 430                  */
 431                 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
 432         }
 433 }
 434
 435 static unsigned int min_odd(unsigned int x, unsigned int y)
 436 {
 437         unsigned int val = min(x, y);
 438
 439         return val % 2 ? val : val - 1;
 440 }
 441
 442 static void result(const char *err, unsigned int n, unsigned int src_off,
 443                    unsigned int dst_off, unsigned int len, unsigned long data)
 444 {
 445         pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 446                 current->comm, n, err, src_off, dst_off, len, data);
 447 }
 448
 449 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 450                        unsigned int dst_off, unsigned int len,
 451                        unsigned long data)
 452 {
 453         pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 454                  current->comm, n, err, src_off, dst_off, len, data);
 455 }
 456
 457 #define verbose_result(err, n, src_off, dst_off, len, data) ({  \
 458         if (verbose)                                            \
 459                 result(err, n, src_off, dst_off, len, data);    \
 460         else                                                    \
 461                 dbg_result(err, n, src_off, dst_off, len, data);\
 462 })
 463
 464 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 465 {
 466         unsigned long long per_sec = 1000000;
 467
 468         if (runtime <= 0)
 469                 return 0;
 470
 471         /* drop precision until runtime is 32-bits */
 472         while (runtime > UINT_MAX) {
 473                 runtime >>= 1;
 474                 per_sec <<= 1;
 475         }
 476
 477         per_sec *= val;
 478         per_sec = INT_TO_FIXPT(per_sec);
 479         do_div(per_sec, runtime);
 480
 481         return per_sec;
 482 }
 483
 484 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 485 {
 486         return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
 487 }
 488
 489 static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
 490 {
 491         unsigned int i;
 492
 493         for (i = 0; i < cnt; i++)
 494                 kfree(d->raw[i]);
 495
 496         kfree(d->aligned);
 497         kfree(d->raw);
 498 }
 499
 500 static void dmatest_free_test_data(struct dmatest_data *d)
 501 {
 502         __dmatest_free_test_data(d, d->cnt);
 503 }
 504
 505 static int dmatest_alloc_test_data(struct dmatest_data *d,
 506                 unsigned int buf_size, u8 align)
 507 {
 508         unsigned int i = 0;
 509
 510         d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 511         if (!d->raw)
 512                 return -ENOMEM;
 513
 514         d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 515         if (!d->aligned)
 516                 goto err;
 517
 518         for (i = 0; i < d->cnt; i++) {
 519                 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
 520                 if (!d->raw[i])
 521                         goto err;
 522
 523                 /* align to alignment restriction */
 524                 if (align)
 525                         d->aligned[i] = PTR_ALIGN(d->raw[i], align);
 526                 else
 527                         d->aligned[i] = d->raw[i];
 528         }
 529
 530         return 0;
 531 err:
 532         __dmatest_free_test_data(d, i);
 533         return -ENOMEM;
 534 }
 535
 536 /*
 537  * This function repeatedly tests DMA transfers of various lengths and
 538  * offsets for a given operation type until it is told to exit by
 539  * kthread_stop(). There may be multiple threads running this function
 540  * in parallel for a single channel, and there may be multiple channels
 541  * being tested in parallel.
 542  *
 543  * Before each test, the source and destination buffer is initialized
 544  * with a known pattern. This pattern is different depending on
 545  * whether it's in an area which is supposed to be copied or
 546  * overwritten, and different in the source and destination buffers.
 547  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 548  * we'll notice.
 549  */
 550 static int dmatest_func(void *data)
 551 {
 552         struct dmatest_thread   *thread = data;
 553         struct dmatest_done     *done = &thread->test_done;
 554         struct dmatest_info     *info;
 555         struct dmatest_params   *params;
 556         struct dma_chan         *chan;
 557         struct dma_device       *dev;
 558         unsigned int            error_count;
 559         unsigned int            failed_tests = 0;
 560         unsigned int            total_tests = 0;
 561         dma_cookie_t            cookie;
 562         enum dma_status         status;
 563         enum dma_ctrl_flags     flags;
 564         u8                      *pq_coefs = NULL;
 565         int                     ret;
 566         unsigned int            buf_size;
 567         struct dmatest_data     *src;
 568         struct dmatest_data     *dst;
 569         int                     i;
 570         ktime_t                 ktime, start, diff;
 571         ktime_t                 filltime = 0;
 572         ktime_t                 comparetime = 0;
 573         s64                     runtime = 0;
 574         unsigned long long      total_len = 0;
 575         unsigned long long      iops = 0;
 576         u8                      align = 0;
 577         bool                    is_memset = false;
 578         dma_addr_t              *srcs;
 579         dma_addr_t              *dma_pq;
 580
 581         set_freezable();
 582
 583         ret = -ENOMEM;
 584
 585         smp_rmb();
 586         thread->pending = false;
 587         info = thread->info;
 588         params = &info->params;
 589         chan = thread->chan;
 590         dev = chan->device;
 591         src = &thread->src;
 592         dst = &thread->dst;
 593         if (thread->type == DMA_MEMCPY) {
 594                 align = params->alignment < 0 ? dev->copy_align :
 595                                                 params->alignment;
 596                 src->cnt = dst->cnt = 1;
 597         } else if (thread->type == DMA_MEMSET) {
 598                 align = params->alignment < 0 ? dev->fill_align :
 599                                                 params->alignment;
 600                 src->cnt = dst->cnt = 1;
 601                 is_memset = true;
 602         } else if (thread->type == DMA_XOR) {
 603                 /* force odd to ensure dst = src */
 604                 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 605                 dst->cnt = 1;
 606                 align = params->alignment < 0 ? dev->xor_align :
 607                                                 params->alignment;
 608         } else if (thread->type == DMA_PQ) {
 609                 /* force odd to ensure dst = src */
 610                 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 611                 dst->cnt = 2;
 612                 align = params->alignment < 0 ? dev->pq_align :
 613                                                 params->alignment;
 614
 615                 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
 616                 if (!pq_coefs)
 617                         goto err_thread_type;
 618
 619                 for (i = 0; i < src->cnt; i++)
 620                         pq_coefs[i] = 1;
 621         } else
 622                 goto err_thread_type;
 623
 624         /* Check if buffer count fits into map count variable (u8) */
 625         if ((src->cnt + dst->cnt) >= 255) {
 626                 pr_err("too many buffers (%d of 255 supported)\n",
 627                        src->cnt + dst->cnt);
 628                 goto err_free_coefs;
 629         }
 630
 631         buf_size = params->buf_size;
 632         if (1 << align > buf_size) {
 633                 pr_err("%u-byte buffer too small for %d-byte alignment\n",
 634                        buf_size, 1 << align);
 635                 goto err_free_coefs;
 636         }
 637
 638         if (dmatest_alloc_test_data(src, buf_size, align) < 0)
 639                 goto err_free_coefs;
 640
 641         if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
 642                 goto err_src;
 643
 644         set_user_nice(current, 10);
 645
 646         srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 647         if (!srcs)
 648                 goto err_dst;
 649
 650         dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 651         if (!dma_pq)
 652                 goto err_srcs_array;
 653
 654         /*
 655          * src and dst buffers are freed by ourselves below
 656          */
 657         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 658
 659         ktime = ktime_get();
 660         while (!kthread_should_stop()
 661                && !(params->iterations && total_tests >= params->iterations)) {
 662                 struct dma_async_tx_descriptor *tx = NULL;
 663                 struct dmaengine_unmap_data *um;
 664                 dma_addr_t *dsts;
 665                 unsigned int len;
 666
 667                 total_tests++;
 668
 669                 if (params->transfer_size) {
 670                         if (params->transfer_size >= buf_size) {
 671                                 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
 672                                        params->transfer_size, buf_size);
 673                                 break;
 674                         }
 675                         len = params->transfer_size;
 676                 } else if (params->norandom) {
 677                         len = buf_size;
 678                 } else {
 679                         len = dmatest_random() % buf_size + 1;
 680                 }
 681
 682                 /* Do not alter transfer size explicitly defined by user */
 683                 if (!params->transfer_size) {
 684                         len = (len >> align) << align;
 685                         if (!len)
 686                                 len = 1 << align;
 687                 }
 688                 total_len += len;
 689
 690                 if (params->norandom) {
 691                         src->off = 0;
 692                         dst->off = 0;
 693                 } else {
 694                         src->off = dmatest_random() % (buf_size - len + 1);
 695                         dst->off = dmatest_random() % (buf_size - len + 1);
 696
 697                         src->off = (src->off >> align) << align;
 698                         dst->off = (dst->off >> align) << align;
 699                 }
 700
 701                 if (!params->noverify) {
 702                         start = ktime_get();
 703                         dmatest_init_srcs(src->aligned, src->off, len,
 704                                           buf_size, is_memset);
 705                         dmatest_init_dsts(dst->aligned, dst->off, len,
 706                                           buf_size, is_memset);
 707
 708                         diff = ktime_sub(ktime_get(), start);
 709                         filltime = ktime_add(filltime, diff);
 710                 }
 711
 712                 um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
 713                                               GFP_KERNEL);
 714                 if (!um) {
 715                         failed_tests++;
 716                         result("unmap data NULL", total_tests,
 717                                src->off, dst->off, len, ret);
 718                         continue;
 719                 }
 720
 721                 um->len = buf_size;
 722                 for (i = 0; i < src->cnt; i++) {
 723                         void *buf = src->aligned[i];
 724                         struct page *pg = virt_to_page(buf);
 725                         unsigned long pg_off = offset_in_page(buf);
 726
 727                         um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
 728                                                    um->len, DMA_TO_DEVICE);
 729                         srcs[i] = um->addr[i] + src->off;
 730                         ret = dma_mapping_error(dev->dev, um->addr[i]);
 731                         if (ret) {
 732                                 result("src mapping error", total_tests,
 733                                        src->off, dst->off, len, ret);
 734                                 goto error_unmap_continue;
 735                         }
 736                         um->to_cnt++;
 737                 }
 738                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 739                 dsts = &um->addr[src->cnt];
 740                 for (i = 0; i < dst->cnt; i++) {
 741                         void *buf = dst->aligned[i];
 742                         struct page *pg = virt_to_page(buf);
 743                         unsigned long pg_off = offset_in_page(buf);
 744
 745                         dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
 746                                                DMA_BIDIRECTIONAL);
 747                         ret = dma_mapping_error(dev->dev, dsts[i]);
 748                         if (ret) {
 749                                 result("dst mapping error", total_tests,
 750                                        src->off, dst->off, len, ret);
 751                                 goto error_unmap_continue;
 752                         }
 753                         um->bidi_cnt++;
 754                 }
 755
 756                 if (thread->type == DMA_MEMCPY)
 757                         tx = dev->device_prep_dma_memcpy(chan,
 758                                                          dsts[0] + dst->off,
 759                                                          srcs[0], len, flags);
 760                 else if (thread->type == DMA_MEMSET)
 761                         tx = dev->device_prep_dma_memset(chan,
 762                                                 dsts[0] + dst->off,
 763                                                 *(src->aligned[0] + src->off),
 764                                                 len, flags);
 765                 else if (thread->type == DMA_XOR)
 766                         tx = dev->device_prep_dma_xor(chan,
 767                                                       dsts[0] + dst->off,
 768                                                       srcs, src->cnt,
 769                                                       len, flags);
 770                 else if (thread->type == DMA_PQ) {
 771                         for (i = 0; i < dst->cnt; i++)
 772                                 dma_pq[i] = dsts[i] + dst->off;
 773                         tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
 774                                                      src->cnt, pq_coefs,
 775                                                      len, flags);
 776                 }
 777
 778                 if (!tx) {
 779                         result("prep error", total_tests, src->off,
 780                                dst->off, len, ret);
 781                         msleep(100);
 782                         goto error_unmap_continue;
 783                 }
 784
 785                 done->done = false;
 786                 tx->callback = dmatest_callback;
 787                 tx->callback_param = done;
 788                 cookie = tx->tx_submit(tx);
 789
 790                 if (dma_submit_error(cookie)) {
 791                         result("submit error", total_tests, src->off,
 792                                dst->off, len, ret);
 793                         msleep(100);
 794                         goto error_unmap_continue;
 795                 }
 796                 dma_async_issue_pending(chan);
 797
 798                 wait_event_freezable_timeout(thread->done_wait, done->done,
 799                                              msecs_to_jiffies(params->timeout));
 800
 801                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 802
 803                 if (!done->done) {
 804                         result("test timed out", total_tests, src->off, dst->off,
 805                                len, 0);
 806                         goto error_unmap_continue;
 807                 } else if (status != DMA_COMPLETE) {
 808                         result(status == DMA_ERROR ?
 809                                "completion error status" :
 810                                "completion busy status", total_tests, src->off,
 811                                dst->off, len, ret);
 812                         goto error_unmap_continue;
 813                 }
 814
 815                 dmaengine_unmap_put(um);
 816
 817                 if (params->noverify) {
 818                         verbose_result("test passed", total_tests, src->off,
 819                                        dst->off, len, 0);
 820                         continue;
 821                 }
 822
 823                 start = ktime_get();
 824                 pr_debug("%s: verifying source buffer...\n", current->comm);
 825                 error_count = dmatest_verify(src->aligned, 0, src->off,
 826                                 0, PATTERN_SRC, true, is_memset);
 827                 error_count += dmatest_verify(src->aligned, src->off,
 828                                 src->off + len, src->off,
 829                                 PATTERN_SRC | PATTERN_COPY, true, is_memset);
 830                 error_count += dmatest_verify(src->aligned, src->off + len,
 831                                 buf_size, src->off + len,
 832                                 PATTERN_SRC, true, is_memset);
 833
 834                 pr_debug("%s: verifying dest buffer...\n", current->comm);
 835                 error_count += dmatest_verify(dst->aligned, 0, dst->off,
 836                                 0, PATTERN_DST, false, is_memset);
 837
 838                 error_count += dmatest_verify(dst->aligned, dst->off,
 839                                 dst->off + len, src->off,
 840                                 PATTERN_SRC | PATTERN_COPY, false, is_memset);
 841
 842                 error_count += dmatest_verify(dst->aligned, dst->off + len,
 843                                 buf_size, dst->off + len,
 844                                 PATTERN_DST, false, is_memset);
 845
 846                 diff = ktime_sub(ktime_get(), start);
 847                 comparetime = ktime_add(comparetime, diff);
 848
 849                 if (error_count) {
 850                         result("data error", total_tests, src->off, dst->off,
 851                                len, error_count);
 852                         failed_tests++;
 853                 } else {
 854                         verbose_result("test passed", total_tests, src->off,
 855                                        dst->off, len, 0);
 856                 }
 857
 858                 continue;
 859
 860 error_unmap_continue:
 861                 dmaengine_unmap_put(um);
 862                 failed_tests++;
 863         }
 864         ktime = ktime_sub(ktime_get(), ktime);
 865         ktime = ktime_sub(ktime, comparetime);
 866         ktime = ktime_sub(ktime, filltime);
 867         runtime = ktime_to_us(ktime);
 868
 869         ret = 0;
 870         kfree(dma_pq);
 871 err_srcs_array:
 872         kfree(srcs);
 873 err_dst:
 874         dmatest_free_test_data(dst);
 875 err_src:
 876         dmatest_free_test_data(src);
 877 err_free_coefs:
 878         kfree(pq_coefs);
 879 err_thread_type:
 880         iops = dmatest_persec(runtime, total_tests);
 881         pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
 882                 current->comm, total_tests, failed_tests,
 883                 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
 884                 dmatest_KBs(runtime, total_len), ret);
 885
 886         /* terminate all transfers on specified channels */
 887         if (ret || failed_tests)
 888                 dmaengine_terminate_sync(chan);
 889
 890         thread->done = true;
 891         wake_up(&thread_wait);
 892
 893         return ret;
 894 }
 895
 896 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 897 {
 898         struct dmatest_thread   *thread;
 899         struct dmatest_thread   *_thread;
 900         int                     ret;
 901
 902         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 903                 ret = kthread_stop(thread->task);
 904                 pr_debug("thread %s exited with status %d\n",
 905                          thread->task->comm, ret);
 906                 list_del(&thread->node);
 907                 put_task_struct(thread->task);
 908                 kfree(thread);
 909         }
 910
 911         /* terminate all transfers on specified channels */
 912         dmaengine_terminate_sync(dtc->chan);
 913
 914         kfree(dtc);
 915 }
 916
 917 static int dmatest_add_threads(struct dmatest_info *info,
 918                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 919 {
 920         struct dmatest_params *params = &info->params;
 921         struct dmatest_thread *thread;
 922         struct dma_chan *chan = dtc->chan;
 923         char *op;
 924         unsigned int i;
 925
 926         if (type == DMA_MEMCPY)
 927                 op = "copy";
 928         else if (type == DMA_MEMSET)
 929                 op = "set";
 930         else if (type == DMA_XOR)
 931                 op = "xor";
 932         else if (type == DMA_PQ)
 933                 op = "pq";
 934         else
 935                 return -EINVAL;
 936
 937         for (i = 0; i < params->threads_per_chan; i++) {
 938                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 939                 if (!thread) {
 940                         pr_warn("No memory for %s-%s%u\n",
 941                                 dma_chan_name(chan), op, i);
 942                         break;
 943                 }
 944                 thread->info = info;
 945                 thread->chan = dtc->chan;
 946                 thread->type = type;
 947                 thread->test_done.wait = &thread->done_wait;
 948                 init_waitqueue_head(&thread->done_wait);
 949                 smp_wmb();
 950                 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
 951                                 dma_chan_name(chan), op, i);
 952                 if (IS_ERR(thread->task)) {
 953                         pr_warn("Failed to create thread %s-%s%u\n",
 954                                 dma_chan_name(chan), op, i);
 955                         kfree(thread);
 956                         break;
 957                 }
 958
 959                 /* srcbuf and dstbuf are allocated by the thread itself */
 960                 get_task_struct(thread->task);
 961                 list_add_tail(&thread->node, &dtc->threads);
 962                 thread->pending = true;
 963         }
 964
 965         return i;
 966 }
 967
 968 static int dmatest_add_channel(struct dmatest_info *info,
 969                 struct dma_chan *chan)
 970 {
 971         struct dmatest_chan     *dtc;
 972         struct dma_device       *dma_dev = chan->device;
 973         unsigned int            thread_count = 0;
 974         int cnt;
 975
 976         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 977         if (!dtc) {
 978                 pr_warn("No memory for %s\n", dma_chan_name(chan));
 979                 return -ENOMEM;
 980         }
 981
 982         dtc->chan = chan;
 983         INIT_LIST_HEAD(&dtc->threads);
 984
 985         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 986                 if (dmatest == 0) {
 987                         cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 988                         thread_count += cnt > 0 ? cnt : 0;
 989                 }
 990         }
 991
 992         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
 993                 if (dmatest == 1) {
 994                         cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
 995                         thread_count += cnt > 0 ? cnt : 0;
 996                 }
 997         }
 998
 999         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1000                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1001                 thread_count += cnt > 0 ? cnt : 0;
1002         }
1003         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1004                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1005                 thread_count += cnt > 0 ? cnt : 0;
1006         }
1007
1008         pr_info("Added %u threads using %s\n",
1009                 thread_count, dma_chan_name(chan));
1010
1011         list_add_tail(&dtc->node, &info->channels);
1012         info->nr_channels++;
1013
1014         return 0;
1015 }
1016
1017 static bool filter(struct dma_chan *chan, void *param)
1018 {
1019         struct dmatest_params *params = param;
1020
1021         if (!dmatest_match_channel(params, chan) ||
1022             !dmatest_match_device(params, chan->device))
1023                 return false;
1024         else
1025                 return true;
1026 }
1027
1028 static void request_channels(struct dmatest_info *info,
1029                              enum dma_transaction_type type)
1030 {
1031         dma_cap_mask_t mask;
1032
1033         dma_cap_zero(mask);
1034         dma_cap_set(type, mask);
1035         for (;;) {
1036                 struct dmatest_params *params = &info->params;
1037                 struct dma_chan *chan;
1038
1039                 chan = dma_request_channel(mask, filter, params);
1040                 if (chan) {
1041                         if (dmatest_add_channel(info, chan)) {
1042                                 dma_release_channel(chan);
1043                                 break; /* add_channel failed, punt */
1044                         }
1045                 } else
1046                         break; /* no more channels available */
1047                 if (params->max_channels &&
1048                     info->nr_channels >= params->max_channels)
1049                         break; /* we have all we need */
1050         }
1051 }
1052
1053 static void add_threaded_test(struct dmatest_info *info)
1054 {
1055         struct dmatest_params *params = &info->params;
1056
1057         /* Copy test parameters */
1058         params->buf_size = test_buf_size;
1059         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1060         strlcpy(params->device, strim(test_device), sizeof(params->device));
1061         params->threads_per_chan = threads_per_chan;
1062         params->max_channels = max_channels;
1063         params->iterations = iterations;
1064         params->xor_sources = xor_sources;
1065         params->pq_sources = pq_sources;
1066         params->timeout = timeout;
1067         params->noverify = noverify;
1068         params->norandom = norandom;
1069         params->alignment = alignment;
1070         params->transfer_size = transfer_size;
1071
1072         request_channels(info, DMA_MEMCPY);
1073         request_channels(info, DMA_MEMSET);
1074         request_channels(info, DMA_XOR);
1075         request_channels(info, DMA_PQ);
1076 }
1077
1078 static void run_pending_tests(struct dmatest_info *info)
1079 {
1080         struct dmatest_chan *dtc;
1081         unsigned int thread_count = 0;
1082
1083         list_for_each_entry(dtc, &info->channels, node) {
1084                 struct dmatest_thread *thread;
1085
1086                 thread_count = 0;
1087                 list_for_each_entry(thread, &dtc->threads, node) {
1088                         wake_up_process(thread->task);
1089                         thread_count++;
1090                 }
1091                 pr_info("Started %u threads using %s\n",
1092                         thread_count, dma_chan_name(dtc->chan));
1093         }
1094 }
1095
1096 static void stop_threaded_test(struct dmatest_info *info)
1097 {
1098         struct dmatest_chan *dtc, *_dtc;
1099         struct dma_chan *chan;
1100
1101         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1102                 list_del(&dtc->node);
1103                 chan = dtc->chan;
1104                 dmatest_cleanup_channel(dtc);
1105                 pr_debug("dropped channel %s\n", dma_chan_name(chan));
1106                 dma_release_channel(chan);
1107         }
1108
1109         info->nr_channels = 0;
1110 }
1111
1112 static void start_threaded_tests(struct dmatest_info *info)
1113 {
1114         /* we might be called early to set run=, defer running until all
1115          * parameters have been evaluated
1116          */
1117         if (!info->did_init)
1118                 return;
1119
1120         run_pending_tests(info);
1121 }
1122
1123 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1124 {
1125         struct dmatest_info *info = &test_info;
1126
1127         mutex_lock(&info->lock);
1128         if (is_threaded_test_run(info)) {
1129                 dmatest_run = true;
1130         } else {
1131                 if (!is_threaded_test_pending(info))
1132                         stop_threaded_test(info);
1133                 dmatest_run = false;
1134         }
1135         mutex_unlock(&info->lock);
1136
1137         return param_get_bool(val, kp);
1138 }
1139
1140 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1141 {
1142         struct dmatest_info *info = &test_info;
1143         int ret;
1144
1145         mutex_lock(&info->lock);
1146         ret = param_set_bool(val, kp);
1147         if (ret) {
1148                 mutex_unlock(&info->lock);
1149                 return ret;
1150         } else if (dmatest_run) {
1151                 if (is_threaded_test_pending(info))
1152                         start_threaded_tests(info);
1153                 else
1154                         pr_info("Could not start test, no channels configured\n");
1155         } else {
1156                 stop_threaded_test(info);
1157         }
1158
1159         mutex_unlock(&info->lock);
1160
1161         return ret;
1162 }
1163
1164 static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1165 {
1166         struct dmatest_info *info = &test_info;
1167         struct dmatest_chan *dtc;
1168         char chan_reset_val[20];
1169         int ret = 0;
1170
1171         mutex_lock(&info->lock);
1172         ret = param_set_copystring(val, kp);
1173         if (ret) {
1174                 mutex_unlock(&info->lock);
1175                 return ret;
1176         }
1177         /*Clear any previously run threads */
1178         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1179                 stop_threaded_test(info);
1180         /* Reject channels that are already registered */
1181         if (is_threaded_test_pending(info)) {
1182                 list_for_each_entry(dtc, &info->channels, node) {
1183                         if (strcmp(dma_chan_name(dtc->chan),
1184                                    strim(test_channel)) == 0) {
1185                                 dtc = list_last_entry(&info->channels,
1186                                                       struct dmatest_chan,
1187                                                       node);
1188                                 strlcpy(chan_reset_val,
1189                                         dma_chan_name(dtc->chan),
1190                                         sizeof(chan_reset_val));
1191                                 ret = -EBUSY;
1192                                 goto add_chan_err;
1193                         }
1194                 }
1195         }
1196
1197         add_threaded_test(info);
1198
1199         /* Check if channel was added successfully */
1200         dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1201
1202         if (dtc->chan) {
1203                 /*
1204                  * if new channel was not successfully added, revert the
1205                  * "test_channel" string to the name of the last successfully
1206                  * added channel. exception for when users issues empty string
1207                  * to channel parameter.
1208                  */
1209                 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1210                     && (strcmp("", strim(test_channel)) != 0)) {
1211                         ret = -EINVAL;
1212                         strlcpy(chan_reset_val, dma_chan_name(dtc->chan),
1213                                 sizeof(chan_reset_val));
1214                         goto add_chan_err;
1215                 }
1216
1217         } else {
1218                 /* Clear test_channel if no channels were added successfully */
1219                 strlcpy(chan_reset_val, "", sizeof(chan_reset_val));
1220                 ret = -EBUSY;
1221                 goto add_chan_err;
1222         }
1223
1224         mutex_unlock(&info->lock);
1225
1226         return ret;
1227
1228 add_chan_err:
1229         param_set_copystring(chan_reset_val, kp);
1230         mutex_unlock(&info->lock);
1231
1232         return ret;
1233 }
1234
1235 static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1236 {
1237         struct dmatest_info *info = &test_info;
1238
1239         mutex_lock(&info->lock);
1240         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1241                 stop_threaded_test(info);
1242                 strlcpy(test_channel, "", sizeof(test_channel));
1243         }
1244         mutex_unlock(&info->lock);
1245
1246         return param_get_string(val, kp);
1247 }
1248
1249 static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1250 {
1251         struct dmatest_info *info = &test_info;
1252         struct dmatest_chan *dtc;
1253         unsigned int thread_count = 0;
1254
1255         list_for_each_entry(dtc, &info->channels, node) {
1256                 struct dmatest_thread *thread;
1257
1258                 thread_count = 0;
1259                 list_for_each_entry(thread, &dtc->threads, node) {
1260                         thread_count++;
1261                 }
1262                 pr_info("%u threads using %s\n",
1263                         thread_count, dma_chan_name(dtc->chan));
1264         }
1265
1266         return 0;
1267 }
1268
1269 static int __init dmatest_init(void)
1270 {
1271         struct dmatest_info *info = &test_info;
1272         struct dmatest_params *params = &info->params;
1273
1274         if (dmatest_run) {
1275                 mutex_lock(&info->lock);
1276                 add_threaded_test(info);
1277                 run_pending_tests(info);
1278                 mutex_unlock(&info->lock);
1279         }
1280
1281         if (params->iterations && wait)
1282                 wait_event(thread_wait, !is_threaded_test_run(info));
1283
1284         /* module parameters are stable, inittime tests are started,
1285          * let userspace take over 'run' control
1286          */
1287         info->did_init = true;
1288
1289         return 0;
1290 }
1291 /* when compiled-in wait for drivers to load first */
1292 late_initcall(dmatest_init);
1293
1294 static void __exit dmatest_exit(void)
1295 {
1296         struct dmatest_info *info = &test_info;
1297
1298         mutex_lock(&info->lock);
1299         stop_threaded_test(info);
1300         mutex_unlock(&info->lock);
1301 }
1302 module_exit(dmatest_exit);
1303
1304 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1305 MODULE_LICENSE("GPL v2");