1
2
3
4
5
6
7
8
9
10
11
12#include <linux/delay.h>
13#include <linux/dmaengine.h>
14#include <linux/init.h>
15#include <linux/kthread.h>
16#include <linux/module.h>
17#include <linux/random.h>
18#include <linux/slab.h>
19#include <linux/of_dma.h>
20#include <linux/platform_device.h>
21#include <linux/wait.h>
22#include <linux/dma/xilinx_dma.h>
23
24static unsigned int test_buf_size = 64;
25module_param(test_buf_size, uint, S_IRUGO);
26MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
27
28static char test_channel[20];
29module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
30MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
31
32static char test_device[20];
33module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
34MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
35
36static unsigned int threads_per_chan = 1;
37module_param(threads_per_chan, uint, S_IRUGO);
38MODULE_PARM_DESC(threads_per_chan,
39 "Number of threads to start per channel (default: 1)");
40
41static unsigned int max_channels;
42module_param(max_channels, uint, S_IRUGO);
43MODULE_PARM_DESC(max_channels,
44 "Maximum number of channels to use (default: all)");
45
46static unsigned int iterations = 5;
47module_param(iterations, uint, S_IRUGO);
48MODULE_PARM_DESC(iterations,
49 "Iterations before stopping test (default: infinite)");
50
51static unsigned int xor_sources = 3;
52module_param(xor_sources, uint, S_IRUGO);
53MODULE_PARM_DESC(xor_sources,
54 "Number of xor source buffers (default: 3)");
55
56static unsigned int pq_sources = 3;
57module_param(pq_sources, uint, S_IRUGO);
58MODULE_PARM_DESC(pq_sources,
59 "Number of p+q source buffers (default: 3)");
60
61
62
63
64
65
66
67
68
69
70
71
72#define PATTERN_SRC 0x80
73#define PATTERN_DST 0x00
74#define PATTERN_COPY 0x40
75#define PATTERN_OVERWRITE 0x20
76#define PATTERN_COUNT_MASK 0x1f
77
78struct cdmatest_thread {
79 struct list_head node;
80 struct task_struct *task;
81 struct dma_chan *chan;
82 u8 **srcs;
83 u8 **dsts;
84 enum dma_transaction_type type;
85 bool done;
86};
87
88struct cdmatest_chan {
89 struct list_head node;
90 struct dma_chan *chan;
91 struct list_head threads;
92};
93
94
95
96
97
98static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
99static LIST_HEAD(cdmatest_channels);
100static unsigned int nr_channels;
101
102static bool is_threaded_test_run(struct cdmatest_chan *tx_dtc)
103{
104 struct cdmatest_thread *thread;
105
106 list_for_each_entry(thread, &tx_dtc->threads, node) {
107 if (!thread->done)
108 return true;
109 }
110
111 return false;
112}
113
114static unsigned long cdmatest_random(void)
115{
116 unsigned long buf;
117
118 get_random_bytes(&buf, sizeof(buf));
119 return buf;
120}
121
122static void cdmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len)
123{
124 unsigned int i;
125 u8 *buf;
126
127 for (; (buf = *bufs); bufs++) {
128 for (i = 0; i < start; i++)
129 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
130 for ( ; i < start + len; i++)
131 buf[i] = PATTERN_SRC | PATTERN_COPY
132 | (~i & PATTERN_COUNT_MASK);
133 for ( ; i < test_buf_size; i++)
134 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
135 buf++;
136 }
137}
138
139static void cdmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len)
140{
141 unsigned int i;
142 u8 *buf;
143
144 for (; (buf = *bufs); bufs++) {
145 for (i = 0; i < start; i++)
146 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
147 for ( ; i < start + len; i++)
148 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
149 | (~i & PATTERN_COUNT_MASK);
150 for ( ; i < test_buf_size; i++)
151 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
152 }
153}
154
155static void cdmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
156 unsigned int counter, bool is_srcbuf)
157{
158 u8 diff = actual ^ pattern;
159 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
160 const char *thread_name = current->comm;
161
162 if (is_srcbuf)
163 pr_warn(
164 "%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
165 thread_name, index, expected, actual);
166 else if ((pattern & PATTERN_COPY)
167 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
168 pr_warn(
169 "%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
170 thread_name, index, expected, actual);
171 else if (diff & PATTERN_SRC)
172 pr_warn(
173 "%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
174 thread_name, index, expected, actual);
175 else
176 pr_warn(
177 "%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
178 thread_name, index, expected, actual);
179}
180
181static unsigned int cdmatest_verify(u8 **bufs, unsigned int start,
182 unsigned int end, unsigned int counter, u8 pattern,
183 bool is_srcbuf)
184{
185 unsigned int i;
186 unsigned int error_count = 0;
187 u8 actual;
188 u8 expected;
189 u8 *buf;
190 unsigned int counter_orig = counter;
191
192 for (; (buf = *bufs); bufs++) {
193 counter = counter_orig;
194 for (i = start; i < end; i++) {
195 actual = buf[i];
196 expected = pattern | (~counter & PATTERN_COUNT_MASK);
197 if (actual != expected) {
198 if (error_count < 32)
199 cdmatest_mismatch(actual, pattern, i,
200 counter, is_srcbuf);
201 error_count++;
202 }
203 counter++;
204 }
205 }
206
207 if (error_count > 32)
208 pr_warn("%s: %u errors suppressed\n",
209 current->comm, error_count - 32);
210
211 return error_count;
212}
213
214static void cdmatest_callback(void *completion)
215{
216 complete(completion);
217}
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233static int cdmatest_func(void *data)
234{
235 struct cdmatest_thread *thread = data;
236 struct dma_chan *chan;
237 const char *thread_name;
238 unsigned int src_off, dst_off, len;
239 unsigned int error_count;
240 unsigned int failed_tests = 0;
241 unsigned int total_tests = 0;
242 dma_cookie_t cookie;
243 enum dma_status status;
244 enum dma_ctrl_flags flags;
245 u8 pq_coefs[pq_sources + 1];
246 int ret;
247 int src_cnt;
248 int dst_cnt;
249 int i;
250
251 thread_name = current->comm;
252
253 ret = -ENOMEM;
254
255
256
257 smp_rmb();
258 chan = thread->chan;
259 if (thread->type == DMA_MEMCPY)
260 src_cnt = dst_cnt = 1;
261 else if (thread->type == DMA_XOR) {
262 src_cnt = xor_sources | 1;
263
264 dst_cnt = 1;
265 } else if (thread->type == DMA_PQ) {
266 src_cnt = pq_sources | 1;
267
268 dst_cnt = 2;
269 for (i = 0; i < src_cnt; i++)
270 pq_coefs[i] = 1;
271 } else
272 goto err_srcs;
273
274 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
275 if (!thread->srcs)
276 goto err_srcs;
277 for (i = 0; i < src_cnt; i++) {
278 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL);
279 if (!thread->srcs[i])
280 goto err_srcbuf;
281 }
282 thread->srcs[i] = NULL;
283
284 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
285 if (!thread->dsts)
286 goto err_dsts;
287 for (i = 0; i < dst_cnt; i++) {
288 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL);
289 if (!thread->dsts[i])
290 goto err_dstbuf;
291 }
292 thread->dsts[i] = NULL;
293
294 set_user_nice(current, 10);
295
296 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
297
298 while (!kthread_should_stop()
299 && !(iterations && total_tests >= iterations)) {
300 struct dma_device *dev = chan->device;
301 struct dma_async_tx_descriptor *tx = NULL;
302 dma_addr_t dma_srcs[src_cnt];
303 dma_addr_t dma_dsts[dst_cnt];
304 struct completion cmp;
305 unsigned long tmo = msecs_to_jiffies(3000);
306 u8 align = 0;
307
308 total_tests++;
309
310
311 if (thread->type == DMA_MEMCPY)
312 align = dev->copy_align;
313 else if (thread->type == DMA_XOR)
314 align = dev->xor_align;
315 else if (thread->type == DMA_PQ)
316 align = dev->pq_align;
317
318 if (1 << align > test_buf_size) {
319 pr_err("%u-byte buffer too small for %d-byte alignment\n",
320 test_buf_size, 1 << align);
321 break;
322 }
323
324 len = cdmatest_random() % test_buf_size + 1;
325 len = (len >> align) << align;
326 if (!len)
327 len = 1 << align;
328 src_off = cdmatest_random() % (test_buf_size - len + 1);
329 dst_off = cdmatest_random() % (test_buf_size - len + 1);
330
331 src_off = (src_off >> align) << align;
332 dst_off = (dst_off >> align) << align;
333
334 cdmatest_init_srcs(thread->srcs, src_off, len);
335 cdmatest_init_dsts(thread->dsts, dst_off, len);
336
337 for (i = 0; i < src_cnt; i++) {
338 u8 *buf = thread->srcs[i] + src_off;
339
340 dma_srcs[i] = dma_map_single(dev->dev, buf, len,
341 DMA_MEM_TO_DEV);
342 }
343
344 for (i = 0; i < dst_cnt; i++) {
345 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
346 test_buf_size,
347 DMA_MEM_TO_MEM);
348 }
349
350 if (thread->type == DMA_MEMCPY) {
351 tx = dev->device_prep_dma_memcpy(chan,
352 dma_dsts[0] + dst_off,
353 dma_srcs[0], len,
354 flags);
355
356 } else if (thread->type == DMA_XOR)
357 tx = dev->device_prep_dma_xor(chan,
358 dma_dsts[0] + dst_off,
359 dma_srcs, src_cnt,
360 len, flags);
361 else if (thread->type == DMA_PQ) {
362 dma_addr_t dma_pq[dst_cnt];
363
364 for (i = 0; i < dst_cnt; i++)
365 dma_pq[i] = dma_dsts[i] + dst_off;
366 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
367 src_cnt, pq_coefs,
368 len, flags);
369 }
370
371 if (!tx) {
372 for (i = 0; i < src_cnt; i++)
373 dma_unmap_single(dev->dev, dma_srcs[i], len,
374 DMA_MEM_TO_DEV);
375 for (i = 0; i < dst_cnt; i++)
376 dma_unmap_single(dev->dev, dma_dsts[i],
377 test_buf_size,
378 DMA_MEM_TO_MEM);
379 pr_warn(
380 "%s: #%u: prep error with src_off=0x%x ",
381 thread_name, total_tests - 1, src_off);
382 pr_warn("dst_off=0x%x len=0x%x\n",
383 dst_off, len);
384 msleep(100);
385 failed_tests++;
386 continue;
387 }
388
389 init_completion(&cmp);
390 tx->callback = cdmatest_callback;
391 tx->callback_param = &cmp;
392 cookie = tx->tx_submit(tx);
393
394 if (dma_submit_error(cookie)) {
395 pr_warn(
396 "%s: #%u: submit error %d with src_off=0x%x ",
397 thread_name, total_tests - 1,
398 cookie, src_off);
399 pr_warn("dst_off=0x%x len=0x%x\n",
400 dst_off, len);
401 msleep(100);
402 failed_tests++;
403 continue;
404 }
405 dma_async_issue_pending(chan);
406
407 tmo = wait_for_completion_timeout(&cmp, tmo);
408 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
409
410 if (tmo == 0) {
411 pr_warn("%s: #%u: test timed out\n",
412 thread_name, total_tests - 1);
413 failed_tests++;
414 continue;
415 } else if (status != DMA_COMPLETE) {
416 pr_warn(
417 "%s: #%u: got completion callback, ",
418 thread_name, total_tests - 1);
419 pr_warn("but status is \'%s\'\n",
420 status == DMA_ERROR ? "error" :
421 "in progress");
422 failed_tests++;
423 continue;
424 }
425
426
427 for (i = 0; i < dst_cnt; i++)
428 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size,
429 DMA_MEM_TO_MEM);
430
431 error_count = 0;
432
433 pr_debug("%s: verifying source buffer...\n", thread_name);
434 error_count += cdmatest_verify(thread->srcs, 0, src_off,
435 0, PATTERN_SRC, true);
436 error_count += cdmatest_verify(thread->srcs, src_off,
437 src_off + len, src_off,
438 PATTERN_SRC | PATTERN_COPY, true);
439 error_count += cdmatest_verify(thread->srcs, src_off + len,
440 test_buf_size, src_off + len,
441 PATTERN_SRC, true);
442
443 pr_debug("%s: verifying dest buffer...\n",
444 thread->task->comm);
445 error_count += cdmatest_verify(thread->dsts, 0, dst_off,
446 0, PATTERN_DST, false);
447 error_count += cdmatest_verify(thread->dsts, dst_off,
448 dst_off + len, src_off,
449 PATTERN_SRC | PATTERN_COPY, false);
450 error_count += cdmatest_verify(thread->dsts, dst_off + len,
451 test_buf_size, dst_off + len,
452 PATTERN_DST, false);
453
454 if (error_count) {
455 pr_warn("%s: #%u: %u errors with ",
456 thread_name, total_tests - 1, error_count);
457 pr_warn("src_off=0x%x dst_off=0x%x len=0x%x\n",
458 src_off, dst_off, len);
459 failed_tests++;
460 } else {
461 pr_debug("%s: #%u: No errors with ",
462 thread_name, total_tests - 1);
463 pr_debug("src_off=0x%x dst_off=0x%x len=0x%x\n",
464 src_off, dst_off, len);
465 }
466 }
467
468 ret = 0;
469 for (i = 0; thread->dsts[i]; i++)
470 kfree(thread->dsts[i]);
471err_dstbuf:
472 kfree(thread->dsts);
473err_dsts:
474 for (i = 0; thread->srcs[i]; i++)
475 kfree(thread->srcs[i]);
476err_srcbuf:
477 kfree(thread->srcs);
478err_srcs:
479 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
480 thread_name, total_tests, failed_tests, ret);
481
482 thread->done = true;
483 wake_up(&thread_wait);
484
485 return ret;
486}
487
488static void cdmatest_cleanup_channel(struct cdmatest_chan *dtc)
489{
490 struct cdmatest_thread *thread;
491 struct cdmatest_thread *_thread;
492 int ret;
493
494 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
495 ret = kthread_stop(thread->task);
496 pr_debug("cdmatest: thread %s exited with status %d\n",
497 thread->task->comm, ret);
498 list_del(&thread->node);
499 put_task_struct(thread->task);
500 kfree(thread);
501 }
502 kfree(dtc);
503}
504
505static int cdmatest_add_threads(struct cdmatest_chan *dtc,
506 enum dma_transaction_type type)
507{
508 struct cdmatest_thread *thread;
509 struct dma_chan *chan = dtc->chan;
510 char *op;
511 unsigned int i;
512
513 if (type == DMA_MEMCPY)
514 op = "copy";
515 else if (type == DMA_XOR)
516 op = "xor";
517 else if (type == DMA_PQ)
518 op = "pq";
519 else
520 return -EINVAL;
521
522 for (i = 0; i < threads_per_chan; i++) {
523 thread = kzalloc(sizeof(struct cdmatest_thread), GFP_KERNEL);
524 if (!thread) {
525 pr_warn("cdmatest: No memory for %s-%s%u\n",
526 dma_chan_name(chan), op, i);
527
528 break;
529 }
530 thread->chan = dtc->chan;
531 thread->type = type;
532 smp_wmb();
533 thread->task = kthread_run(cdmatest_func, thread, "%s-%s%u",
534 dma_chan_name(chan), op, i);
535 if (IS_ERR(thread->task)) {
536 pr_warn("cdmatest: Failed to run thread %s-%s%u\n",
537 dma_chan_name(chan), op, i);
538 kfree(thread);
539 break;
540 }
541
542
543 get_task_struct(thread->task);
544 list_add_tail(&thread->node, &dtc->threads);
545 }
546
547 return i;
548}
549
550static int cdmatest_add_channel(struct dma_chan *chan)
551{
552 struct cdmatest_chan *dtc;
553 struct dma_device *dma_dev = chan->device;
554 unsigned int thread_count = 0;
555 int cnt;
556
557 dtc = kmalloc(sizeof(struct cdmatest_chan), GFP_KERNEL);
558 if (!dtc) {
559 pr_warn("cdmatest: No memory for %s\n", dma_chan_name(chan));
560 return -ENOMEM;
561 }
562
563 dtc->chan = chan;
564 INIT_LIST_HEAD(&dtc->threads);
565
566 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
567 cnt = cdmatest_add_threads(dtc, DMA_MEMCPY);
568 thread_count += cnt > 0 ? cnt : 0;
569 }
570 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
571 cnt = cdmatest_add_threads(dtc, DMA_XOR);
572 thread_count += cnt > 0 ? cnt : 0;
573 }
574 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
575 cnt = cdmatest_add_threads(dtc, DMA_PQ);
576 thread_count += cnt > 0 ? cnt : 0;
577 }
578
579 pr_info("cdmatest: Started %u threads using %s\n",
580 thread_count, dma_chan_name(chan));
581
582 list_add_tail(&dtc->node, &cdmatest_channels);
583 nr_channels++;
584
585 if (iterations)
586 wait_event(thread_wait, !is_threaded_test_run(dtc));
587
588 return 0;
589}
590
591static int xilinx_cdmatest_probe(struct platform_device *pdev)
592{
593 struct dma_chan *chan;
594 int err;
595
596 chan = dma_request_slave_channel(&pdev->dev, "cdma");
597 if (IS_ERR(chan)) {
598 pr_err("xilinx_cdmatest: No channel\n");
599 return PTR_ERR(chan);
600 }
601
602 err = cdmatest_add_channel(chan);
603 if (err) {
604 pr_err("xilinx_cdmatest: Unable to add channel\n");
605 goto free_tx;
606 }
607 return 0;
608
609free_tx:
610 dma_release_channel(chan);
611
612 return err;
613}
614
615static int xilinx_cdmatest_remove(struct platform_device *pdev)
616{
617 struct cdmatest_chan *dtc, *_dtc;
618 struct dma_chan *chan;
619
620 list_for_each_entry_safe(dtc, _dtc, &cdmatest_channels, node) {
621 list_del(&dtc->node);
622 chan = dtc->chan;
623 cdmatest_cleanup_channel(dtc);
624 pr_info("xilinx_cdmatest: dropped channel %s\n",
625 dma_chan_name(chan));
626 dma_release_channel(chan);
627 }
628 return 0;
629}
630
631static const struct of_device_id xilinx_cdmatest_of_ids[] = {
632 { .compatible = "xlnx,axi-cdma-test-1.00.a", },
633 {}
634};
635
636static struct platform_driver xilinx_cdmatest_driver = {
637 .driver = {
638 .name = "xilinx_cdmatest",
639 .owner = THIS_MODULE,
640 .of_match_table = xilinx_cdmatest_of_ids,
641 },
642 .probe = xilinx_cdmatest_probe,
643 .remove = xilinx_cdmatest_remove,
644};
645
646static int __init cdma_init(void)
647{
648 return platform_driver_register(&xilinx_cdmatest_driver);
649
650}
651late_initcall(cdma_init);
652
653static void __exit cdma_exit(void)
654{
655 platform_driver_unregister(&xilinx_cdmatest_driver);
656}
657module_exit(cdma_exit)
658
659MODULE_AUTHOR("Xilinx, Inc.");
660MODULE_DESCRIPTION("Xilinx AXI CDMA Test Client");
661MODULE_LICENSE("GPL v2");
662