1
2
3
4
5
6
7
8
9
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/delay.h>
14#include <linux/dma-mapping.h>
15#include <linux/dmaengine.h>
16#include <linux/freezer.h>
17#include <linux/init.h>
18#include <linux/kthread.h>
19#include <linux/module.h>
20#include <linux/moduleparam.h>
21#include <linux/random.h>
22#include <linux/slab.h>
23#include <linux/wait.h>
24
25static unsigned int test_buf_size = 16384;
26module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
27MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
28
29static char test_channel[20];
30module_param_string(channel, test_channel, sizeof(test_channel),
31 S_IRUGO | S_IWUSR);
32MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
33
34static char test_device[32];
35module_param_string(device, test_device, sizeof(test_device),
36 S_IRUGO | S_IWUSR);
37MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
38
39static unsigned int threads_per_chan = 1;
40module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
41MODULE_PARM_DESC(threads_per_chan,
42 "Number of threads to start per channel (default: 1)");
43
44static unsigned int max_channels;
45module_param(max_channels, uint, S_IRUGO | S_IWUSR);
46MODULE_PARM_DESC(max_channels,
47 "Maximum number of channels to use (default: all)");
48
49static unsigned int iterations;
50module_param(iterations, uint, S_IRUGO | S_IWUSR);
51MODULE_PARM_DESC(iterations,
52 "Iterations before stopping test (default: infinite)");
53
54static unsigned int sg_buffers = 1;
55module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
56MODULE_PARM_DESC(sg_buffers,
57 "Number of scatter gather buffers (default: 1)");
58
59static unsigned int dmatest = 1;
60module_param(dmatest, uint, S_IRUGO | S_IWUSR);
61MODULE_PARM_DESC(dmatest,
62 "dmatest 0-memcpy 1-slave_sg (default: 1)");
63
64static unsigned int xor_sources = 3;
65module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
66MODULE_PARM_DESC(xor_sources,
67 "Number of xor source buffers (default: 3)");
68
69static unsigned int pq_sources = 3;
70module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
71MODULE_PARM_DESC(pq_sources,
72 "Number of p+q source buffers (default: 3)");
73
74static int timeout = 3000;
75module_param(timeout, uint, S_IRUGO | S_IWUSR);
76MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
77 "Pass -1 for infinite timeout");
78
79static bool noverify;
80module_param(noverify, bool, S_IRUGO | S_IWUSR);
81MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
82
83static bool verbose;
84module_param(verbose, bool, S_IRUGO | S_IWUSR);
85MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
86
87
88
89
90
91
92
93
94
95
96
97
98
99struct dmatest_params {
100 unsigned int buf_size;
101 char channel[20];
102 char device[32];
103 unsigned int threads_per_chan;
104 unsigned int max_channels;
105 unsigned int iterations;
106 unsigned int xor_sources;
107 unsigned int pq_sources;
108 int timeout;
109 bool noverify;
110};
111
112
113
114
115
116
117static struct dmatest_info {
118
119 struct dmatest_params params;
120
121
122 struct list_head channels;
123 unsigned int nr_channels;
124 struct mutex lock;
125 bool did_init;
126} test_info = {
127 .channels = LIST_HEAD_INIT(test_info.channels),
128 .lock = __MUTEX_INITIALIZER(test_info.lock),
129};
130
131static int dmatest_run_set(const char *val, const struct kernel_param *kp);
132static int dmatest_run_get(char *val, const struct kernel_param *kp);
133static const struct kernel_param_ops run_ops = {
134 .set = dmatest_run_set,
135 .get = dmatest_run_get,
136};
137static bool dmatest_run;
138module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
139MODULE_PARM_DESC(run, "Run the test (default: false)");
140
141
142#define MAX_ERROR_COUNT 32
143
144
145
146
147
148
149
150
151
152
153
154
155#define PATTERN_SRC 0x80
156#define PATTERN_DST 0x00
157#define PATTERN_COPY 0x40
158#define PATTERN_OVERWRITE 0x20
159#define PATTERN_COUNT_MASK 0x1f
160
161struct dmatest_thread {
162 struct list_head node;
163 struct dmatest_info *info;
164 struct task_struct *task;
165 struct dma_chan *chan;
166 u8 **srcs;
167 u8 **dsts;
168 enum dma_transaction_type type;
169 bool done;
170};
171
172struct dmatest_chan {
173 struct list_head node;
174 struct dma_chan *chan;
175 struct list_head threads;
176};
177
178static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
179static bool wait;
180
181static bool is_threaded_test_run(struct dmatest_info *info)
182{
183 struct dmatest_chan *dtc;
184
185 list_for_each_entry(dtc, &info->channels, node) {
186 struct dmatest_thread *thread;
187
188 list_for_each_entry(thread, &dtc->threads, node) {
189 if (!thread->done)
190 return true;
191 }
192 }
193
194 return false;
195}
196
197static int dmatest_wait_get(char *val, const struct kernel_param *kp)
198{
199 struct dmatest_info *info = &test_info;
200 struct dmatest_params *params = &info->params;
201
202 if (params->iterations)
203 wait_event(thread_wait, !is_threaded_test_run(info));
204 wait = true;
205 return param_get_bool(val, kp);
206}
207
208static const struct kernel_param_ops wait_ops = {
209 .get = dmatest_wait_get,
210 .set = param_set_bool,
211};
212module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
213MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
214
215static bool dmatest_match_channel(struct dmatest_params *params,
216 struct dma_chan *chan)
217{
218 if (params->channel[0] == '\0')
219 return true;
220 return strcmp(dma_chan_name(chan), params->channel) == 0;
221}
222
223static bool dmatest_match_device(struct dmatest_params *params,
224 struct dma_device *device)
225{
226 if (params->device[0] == '\0')
227 return true;
228 return strcmp(dev_name(device->dev), params->device) == 0;
229}
230
231static unsigned long dmatest_random(void)
232{
233 unsigned long buf;
234
235 prandom_bytes(&buf, sizeof(buf));
236 return buf;
237}
238
239static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
240 unsigned int buf_size)
241{
242 unsigned int i;
243 u8 *buf;
244
245 for (; (buf = *bufs); bufs++) {
246 for (i = 0; i < start; i++)
247 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
248 for ( ; i < start + len; i++)
249 buf[i] = PATTERN_SRC | PATTERN_COPY
250 | (~i & PATTERN_COUNT_MASK);
251 for ( ; i < buf_size; i++)
252 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
253 buf++;
254 }
255}
256
257static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
258 unsigned int buf_size)
259{
260 unsigned int i;
261 u8 *buf;
262
263 for (; (buf = *bufs); bufs++) {
264 for (i = 0; i < start; i++)
265 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
266 for ( ; i < start + len; i++)
267 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
268 | (~i & PATTERN_COUNT_MASK);
269 for ( ; i < buf_size; i++)
270 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
271 }
272}
273
274static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
275 unsigned int counter, bool is_srcbuf)
276{
277 u8 diff = actual ^ pattern;
278 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
279 const char *thread_name = current->comm;
280
281 if (is_srcbuf)
282 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
283 thread_name, index, expected, actual);
284 else if ((pattern & PATTERN_COPY)
285 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
286 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
287 thread_name, index, expected, actual);
288 else if (diff & PATTERN_SRC)
289 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
290 thread_name, index, expected, actual);
291 else
292 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
293 thread_name, index, expected, actual);
294}
295
296static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
297 unsigned int end, unsigned int counter, u8 pattern,
298 bool is_srcbuf)
299{
300 unsigned int i;
301 unsigned int error_count = 0;
302 u8 actual;
303 u8 expected;
304 u8 *buf;
305 unsigned int counter_orig = counter;
306
307 for (; (buf = *bufs); bufs++) {
308 counter = counter_orig;
309 for (i = start; i < end; i++) {
310 actual = buf[i];
311 expected = pattern | (~counter & PATTERN_COUNT_MASK);
312 if (actual != expected) {
313 if (error_count < MAX_ERROR_COUNT)
314 dmatest_mismatch(actual, pattern, i,
315 counter, is_srcbuf);
316 error_count++;
317 }
318 counter++;
319 }
320 }
321
322 if (error_count > MAX_ERROR_COUNT)
323 pr_warn("%s: %u errors suppressed\n",
324 current->comm, error_count - MAX_ERROR_COUNT);
325
326 return error_count;
327}
328
329
330struct dmatest_done {
331 bool done;
332 wait_queue_head_t *wait;
333};
334
335static void dmatest_callback(void *arg)
336{
337 struct dmatest_done *done = arg;
338
339 done->done = true;
340 wake_up_all(done->wait);
341}
342
343static unsigned int min_odd(unsigned int x, unsigned int y)
344{
345 unsigned int val = min(x, y);
346
347 return val % 2 ? val : val - 1;
348}
349
350static void result(const char *err, unsigned int n, unsigned int src_off,
351 unsigned int dst_off, unsigned int len, unsigned long data)
352{
353 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
354 current->comm, n, err, src_off, dst_off, len, data);
355}
356
357static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
358 unsigned int dst_off, unsigned int len,
359 unsigned long data)
360{
361 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
362 current->comm, n, err, src_off, dst_off, len, data);
363}
364
365#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
366 if (verbose) \
367 result(err, n, src_off, dst_off, len, data); \
368 else \
369 dbg_result(err, n, src_off, dst_off, len, data);\
370})
371
372static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
373{
374 unsigned long long per_sec = 1000000;
375
376 if (runtime <= 0)
377 return 0;
378
379
380 while (runtime > UINT_MAX) {
381 runtime >>= 1;
382 per_sec <<= 1;
383 }
384
385 per_sec *= val;
386 do_div(per_sec, runtime);
387 return per_sec;
388}
389
390static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
391{
392 return dmatest_persec(runtime, len >> 10);
393}
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409static int dmatest_func(void *data)
410{
411 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
412 struct dmatest_thread *thread = data;
413 struct dmatest_done done = { .wait = &done_wait };
414 struct dmatest_info *info;
415 struct dmatest_params *params;
416 struct dma_chan *chan;
417 struct dma_device *dev;
418 unsigned int error_count;
419 unsigned int failed_tests = 0;
420 unsigned int total_tests = 0;
421 dma_cookie_t cookie;
422 enum dma_status status;
423 enum dma_ctrl_flags flags;
424 u8 *pq_coefs = NULL;
425 int ret;
426 int src_cnt;
427 int dst_cnt;
428 int i;
429 ktime_t ktime;
430 s64 runtime = 0;
431 unsigned long long total_len = 0;
432
433 set_freezable();
434
435 ret = -ENOMEM;
436
437 smp_rmb();
438 info = thread->info;
439 params = &info->params;
440 chan = thread->chan;
441 dev = chan->device;
442 if (thread->type == DMA_MEMCPY)
443 src_cnt = dst_cnt = 1;
444 else if (thread->type == DMA_SG)
445 src_cnt = dst_cnt = sg_buffers;
446 else if (thread->type == DMA_XOR) {
447
448 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
449 dst_cnt = 1;
450 } else if (thread->type == DMA_PQ) {
451
452 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
453 dst_cnt = 2;
454
455 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
456 if (!pq_coefs)
457 goto err_thread_type;
458
459 for (i = 0; i < src_cnt; i++)
460 pq_coefs[i] = 1;
461 } else
462 goto err_thread_type;
463
464 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
465 if (!thread->srcs)
466 goto err_srcs;
467 for (i = 0; i < src_cnt; i++) {
468 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
469 if (!thread->srcs[i])
470 goto err_srcbuf;
471 }
472 thread->srcs[i] = NULL;
473
474 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
475 if (!thread->dsts)
476 goto err_dsts;
477 for (i = 0; i < dst_cnt; i++) {
478 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
479 if (!thread->dsts[i])
480 goto err_dstbuf;
481 }
482 thread->dsts[i] = NULL;
483
484 set_user_nice(current, 10);
485
486
487
488
489 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
490
491 ktime = ktime_get();
492 while (!kthread_should_stop()
493 && !(params->iterations && total_tests >= params->iterations)) {
494 struct dma_async_tx_descriptor *tx = NULL;
495 struct dmaengine_unmap_data *um;
496 dma_addr_t srcs[src_cnt];
497 dma_addr_t *dsts;
498 unsigned int src_off, dst_off, len;
499 u8 align = 0;
500 struct scatterlist tx_sg[src_cnt];
501 struct scatterlist rx_sg[src_cnt];
502
503 total_tests++;
504
505
506 if (thread->type == DMA_MEMCPY)
507 align = dev->copy_align;
508 else if (thread->type == DMA_XOR)
509 align = dev->xor_align;
510 else if (thread->type == DMA_PQ)
511 align = dev->pq_align;
512
513 if (1 << align > params->buf_size) {
514 pr_err("%u-byte buffer too small for %d-byte alignment\n",
515 params->buf_size, 1 << align);
516 break;
517 }
518
519 align = 3;
520
521 if (params->noverify)
522 len = params->buf_size;
523 else
524 len = dmatest_random() % params->buf_size + 1;
525
526 len = (len >> align) << align;
527 if (!len)
528 len = 1 << align;
529
530 total_len += len;
531
532 if (params->noverify) {
533 src_off = 0;
534 dst_off = 0;
535 } else {
536 src_off = dmatest_random() % (params->buf_size - len + 1);
537 dst_off = dmatest_random() % (params->buf_size - len + 1);
538
539 src_off = (src_off >> align) << align;
540 dst_off = (dst_off >> align) << align;
541
542 dmatest_init_srcs(thread->srcs, src_off, len,
543 params->buf_size);
544 dmatest_init_dsts(thread->dsts, dst_off, len,
545 params->buf_size);
546 }
547
548 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
549 GFP_KERNEL);
550 if (!um) {
551 failed_tests++;
552 result("unmap data NULL", total_tests,
553 src_off, dst_off, len, ret);
554 continue;
555 }
556
557 um->len = params->buf_size;
558 for (i = 0; i < src_cnt; i++) {
559 void *buf = thread->srcs[i];
560 struct page *pg = virt_to_page(buf);
561 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
562
563 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
564 um->len, DMA_TO_DEVICE);
565 srcs[i] = um->addr[i] + src_off;
566 ret = dma_mapping_error(dev->dev, um->addr[i]);
567 if (ret) {
568 dmaengine_unmap_put(um);
569 result("src mapping error", total_tests,
570 src_off, dst_off, len, ret);
571 failed_tests++;
572 continue;
573 }
574 um->to_cnt++;
575 }
576
577 dsts = &um->addr[src_cnt];
578 for (i = 0; i < dst_cnt; i++) {
579 void *buf = thread->dsts[i];
580 struct page *pg = virt_to_page(buf);
581 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
582
583 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
584 DMA_BIDIRECTIONAL);
585 ret = dma_mapping_error(dev->dev, dsts[i]);
586 if (ret) {
587 dmaengine_unmap_put(um);
588 result("dst mapping error", total_tests,
589 src_off, dst_off, len, ret);
590 failed_tests++;
591 continue;
592 }
593 um->bidi_cnt++;
594 }
595
596 sg_init_table(tx_sg, src_cnt);
597 sg_init_table(rx_sg, src_cnt);
598 for (i = 0; i < src_cnt; i++) {
599 sg_dma_address(&rx_sg[i]) = srcs[i];
600 sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
601 sg_dma_len(&tx_sg[i]) = len;
602 sg_dma_len(&rx_sg[i]) = len;
603 }
604
605 if (thread->type == DMA_MEMCPY)
606 tx = dev->device_prep_dma_memcpy(chan,
607 dsts[0] + dst_off,
608 srcs[0], len, flags);
609 else if (thread->type == DMA_SG)
610 tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
611 rx_sg, src_cnt, flags);
612 else if (thread->type == DMA_XOR)
613 tx = dev->device_prep_dma_xor(chan,
614 dsts[0] + dst_off,
615 srcs, src_cnt,
616 len, flags);
617 else if (thread->type == DMA_PQ) {
618 dma_addr_t dma_pq[dst_cnt];
619
620 for (i = 0; i < dst_cnt; i++)
621 dma_pq[i] = dsts[i] + dst_off;
622 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
623 src_cnt, pq_coefs,
624 len, flags);
625 }
626
627 if (!tx) {
628 dmaengine_unmap_put(um);
629 result("prep error", total_tests, src_off,
630 dst_off, len, ret);
631 msleep(100);
632 failed_tests++;
633 continue;
634 }
635
636 done.done = false;
637 tx->callback = dmatest_callback;
638 tx->callback_param = &done;
639 cookie = tx->tx_submit(tx);
640
641 if (dma_submit_error(cookie)) {
642 dmaengine_unmap_put(um);
643 result("submit error", total_tests, src_off,
644 dst_off, len, ret);
645 msleep(100);
646 failed_tests++;
647 continue;
648 }
649 dma_async_issue_pending(chan);
650
651 wait_event_freezable_timeout(done_wait, done.done,
652 msecs_to_jiffies(params->timeout));
653
654 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
655
656 if (!done.done) {
657
658
659
660
661
662
663
664
665 dmaengine_unmap_put(um);
666 result("test timed out", total_tests, src_off, dst_off,
667 len, 0);
668 failed_tests++;
669 continue;
670 } else if (status != DMA_COMPLETE) {
671 dmaengine_unmap_put(um);
672 result(status == DMA_ERROR ?
673 "completion error status" :
674 "completion busy status", total_tests, src_off,
675 dst_off, len, ret);
676 failed_tests++;
677 continue;
678 }
679
680 dmaengine_unmap_put(um);
681
682 if (params->noverify) {
683 verbose_result("test passed", total_tests, src_off,
684 dst_off, len, 0);
685 continue;
686 }
687
688 pr_debug("%s: verifying source buffer...\n", current->comm);
689 error_count = dmatest_verify(thread->srcs, 0, src_off,
690 0, PATTERN_SRC, true);
691 error_count += dmatest_verify(thread->srcs, src_off,
692 src_off + len, src_off,
693 PATTERN_SRC | PATTERN_COPY, true);
694 error_count += dmatest_verify(thread->srcs, src_off + len,
695 params->buf_size, src_off + len,
696 PATTERN_SRC, true);
697
698 pr_debug("%s: verifying dest buffer...\n", current->comm);
699 error_count += dmatest_verify(thread->dsts, 0, dst_off,
700 0, PATTERN_DST, false);
701 error_count += dmatest_verify(thread->dsts, dst_off,
702 dst_off + len, src_off,
703 PATTERN_SRC | PATTERN_COPY, false);
704 error_count += dmatest_verify(thread->dsts, dst_off + len,
705 params->buf_size, dst_off + len,
706 PATTERN_DST, false);
707
708 if (error_count) {
709 result("data error", total_tests, src_off, dst_off,
710 len, error_count);
711 failed_tests++;
712 } else {
713 verbose_result("test passed", total_tests, src_off,
714 dst_off, len, 0);
715 }
716 }
717 runtime = ktime_us_delta(ktime_get(), ktime);
718
719 ret = 0;
720err_dstbuf:
721 for (i = 0; thread->dsts[i]; i++)
722 kfree(thread->dsts[i]);
723 kfree(thread->dsts);
724err_dsts:
725err_srcbuf:
726 for (i = 0; thread->srcs[i]; i++)
727 kfree(thread->srcs[i]);
728 kfree(thread->srcs);
729err_srcs:
730 kfree(pq_coefs);
731err_thread_type:
732 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
733 current->comm, total_tests, failed_tests,
734 dmatest_persec(runtime, total_tests),
735 dmatest_KBs(runtime, total_len), ret);
736
737
738 if (ret)
739 dmaengine_terminate_all(chan);
740
741 thread->done = true;
742 wake_up(&thread_wait);
743
744 return ret;
745}
746
747static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
748{
749 struct dmatest_thread *thread;
750 struct dmatest_thread *_thread;
751 int ret;
752
753 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
754 ret = kthread_stop(thread->task);
755 pr_debug("thread %s exited with status %d\n",
756 thread->task->comm, ret);
757 list_del(&thread->node);
758 put_task_struct(thread->task);
759 kfree(thread);
760 }
761
762
763 dmaengine_terminate_all(dtc->chan);
764
765 kfree(dtc);
766}
767
768static int dmatest_add_threads(struct dmatest_info *info,
769 struct dmatest_chan *dtc, enum dma_transaction_type type)
770{
771 struct dmatest_params *params = &info->params;
772 struct dmatest_thread *thread;
773 struct dma_chan *chan = dtc->chan;
774 char *op;
775 unsigned int i;
776
777 if (type == DMA_MEMCPY)
778 op = "copy";
779 else if (type == DMA_SG)
780 op = "sg";
781 else if (type == DMA_XOR)
782 op = "xor";
783 else if (type == DMA_PQ)
784 op = "pq";
785 else
786 return -EINVAL;
787
788 for (i = 0; i < params->threads_per_chan; i++) {
789 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
790 if (!thread) {
791 pr_warn("No memory for %s-%s%u\n",
792 dma_chan_name(chan), op, i);
793 break;
794 }
795 thread->info = info;
796 thread->chan = dtc->chan;
797 thread->type = type;
798 smp_wmb();
799 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
800 dma_chan_name(chan), op, i);
801 if (IS_ERR(thread->task)) {
802 pr_warn("Failed to create thread %s-%s%u\n",
803 dma_chan_name(chan), op, i);
804 kfree(thread);
805 break;
806 }
807
808
809 get_task_struct(thread->task);
810 list_add_tail(&thread->node, &dtc->threads);
811 wake_up_process(thread->task);
812 }
813
814 return i;
815}
816
817static int dmatest_add_channel(struct dmatest_info *info,
818 struct dma_chan *chan)
819{
820 struct dmatest_chan *dtc;
821 struct dma_device *dma_dev = chan->device;
822 unsigned int thread_count = 0;
823 int cnt;
824
825 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
826 if (!dtc) {
827 pr_warn("No memory for %s\n", dma_chan_name(chan));
828 return -ENOMEM;
829 }
830
831 dtc->chan = chan;
832 INIT_LIST_HEAD(&dtc->threads);
833
834 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
835 if (dmatest == 0) {
836 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
837 thread_count += cnt > 0 ? cnt : 0;
838 }
839 }
840
841 if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
842 if (dmatest == 1) {
843 cnt = dmatest_add_threads(info, dtc, DMA_SG);
844 thread_count += cnt > 0 ? cnt : 0;
845 }
846 }
847
848 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
849 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
850 thread_count += cnt > 0 ? cnt : 0;
851 }
852 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
853 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
854 thread_count += cnt > 0 ? cnt : 0;
855 }
856
857 pr_info("Started %u threads using %s\n",
858 thread_count, dma_chan_name(chan));
859
860 list_add_tail(&dtc->node, &info->channels);
861 info->nr_channels++;
862
863 return 0;
864}
865
866static bool filter(struct dma_chan *chan, void *param)
867{
868 struct dmatest_params *params = param;
869
870 if (!dmatest_match_channel(params, chan) ||
871 !dmatest_match_device(params, chan->device))
872 return false;
873 else
874 return true;
875}
876
877static void request_channels(struct dmatest_info *info,
878 enum dma_transaction_type type)
879{
880 dma_cap_mask_t mask;
881
882 dma_cap_zero(mask);
883 dma_cap_set(type, mask);
884 for (;;) {
885 struct dmatest_params *params = &info->params;
886 struct dma_chan *chan;
887
888 chan = dma_request_channel(mask, filter, params);
889 if (chan) {
890 if (dmatest_add_channel(info, chan)) {
891 dma_release_channel(chan);
892 break;
893 }
894 } else
895 break;
896 if (params->max_channels &&
897 info->nr_channels >= params->max_channels)
898 break;
899 }
900}
901
902static void run_threaded_test(struct dmatest_info *info)
903{
904 struct dmatest_params *params = &info->params;
905
906
907 params->buf_size = test_buf_size;
908 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
909 strlcpy(params->device, strim(test_device), sizeof(params->device));
910 params->threads_per_chan = threads_per_chan;
911 params->max_channels = max_channels;
912 params->iterations = iterations;
913 params->xor_sources = xor_sources;
914 params->pq_sources = pq_sources;
915 params->timeout = timeout;
916 params->noverify = noverify;
917
918 request_channels(info, DMA_MEMCPY);
919 request_channels(info, DMA_XOR);
920 request_channels(info, DMA_SG);
921 request_channels(info, DMA_PQ);
922}
923
924static void stop_threaded_test(struct dmatest_info *info)
925{
926 struct dmatest_chan *dtc, *_dtc;
927 struct dma_chan *chan;
928
929 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
930 list_del(&dtc->node);
931 chan = dtc->chan;
932 dmatest_cleanup_channel(dtc);
933 pr_debug("dropped channel %s\n", dma_chan_name(chan));
934 dma_release_channel(chan);
935 }
936
937 info->nr_channels = 0;
938}
939
940static void restart_threaded_test(struct dmatest_info *info, bool run)
941{
942
943
944
945 if (!info->did_init)
946 return;
947
948
949 stop_threaded_test(info);
950
951
952 run_threaded_test(info);
953}
954
955static int dmatest_run_get(char *val, const struct kernel_param *kp)
956{
957 struct dmatest_info *info = &test_info;
958
959 mutex_lock(&info->lock);
960 if (is_threaded_test_run(info)) {
961 dmatest_run = true;
962 } else {
963 stop_threaded_test(info);
964 dmatest_run = false;
965 }
966 mutex_unlock(&info->lock);
967
968 return param_get_bool(val, kp);
969}
970
971static int dmatest_run_set(const char *val, const struct kernel_param *kp)
972{
973 struct dmatest_info *info = &test_info;
974 int ret;
975
976 mutex_lock(&info->lock);
977 ret = param_set_bool(val, kp);
978 if (ret) {
979 mutex_unlock(&info->lock);
980 return ret;
981 }
982
983 if (is_threaded_test_run(info))
984 ret = -EBUSY;
985 else if (dmatest_run)
986 restart_threaded_test(info, dmatest_run);
987
988 mutex_unlock(&info->lock);
989
990 return ret;
991}
992
993static int __init dmatest_init(void)
994{
995 struct dmatest_info *info = &test_info;
996 struct dmatest_params *params = &info->params;
997
998 if (dmatest_run) {
999 mutex_lock(&info->lock);
1000 run_threaded_test(info);
1001 mutex_unlock(&info->lock);
1002 }
1003
1004 if (params->iterations && wait)
1005 wait_event(thread_wait, !is_threaded_test_run(info));
1006
1007
1008
1009
1010 info->did_init = true;
1011
1012 return 0;
1013}
1014
1015late_initcall(dmatest_init);
1016
1017static void __exit dmatest_exit(void)
1018{
1019 struct dmatest_info *info = &test_info;
1020
1021 mutex_lock(&info->lock);
1022 stop_threaded_test(info);
1023 mutex_unlock(&info->lock);
1024}
1025module_exit(dmatest_exit);
1026
1027MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1028MODULE_LICENSE("GPL v2");
1029