linux/drivers/dma/dmatest.c
<<
>>
Prefs
   1/*
   2 * DMA Engine test module
   3 *
   4 * Copyright (C) 2007 Atmel Corporation
   5 * Copyright (C) 2013 Intel Corporation
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13#include <linux/delay.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/dmaengine.h>
  16#include <linux/freezer.h>
  17#include <linux/init.h>
  18#include <linux/kthread.h>
  19#include <linux/module.h>
  20#include <linux/moduleparam.h>
  21#include <linux/random.h>
  22#include <linux/slab.h>
  23#include <linux/wait.h>
  24
  25static unsigned int test_buf_size = 16384;
  26module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  27MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  28
  29static char test_channel[20];
  30module_param_string(channel, test_channel, sizeof(test_channel),
  31                S_IRUGO | S_IWUSR);
  32MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  33
  34static char test_device[32];
  35module_param_string(device, test_device, sizeof(test_device),
  36                S_IRUGO | S_IWUSR);
  37MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39static unsigned int threads_per_chan = 1;
  40module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  41MODULE_PARM_DESC(threads_per_chan,
  42                "Number of threads to start per channel (default: 1)");
  43
  44static unsigned int max_channels;
  45module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  46MODULE_PARM_DESC(max_channels,
  47                "Maximum number of channels to use (default: all)");
  48
  49static unsigned int iterations;
  50module_param(iterations, uint, S_IRUGO | S_IWUSR);
  51MODULE_PARM_DESC(iterations,
  52                "Iterations before stopping test (default: infinite)");
  53
  54static unsigned int sg_buffers = 1;
  55module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
  56MODULE_PARM_DESC(sg_buffers,
  57                "Number of scatter gather buffers (default: 1)");
  58
  59static unsigned int dmatest;
  60module_param(dmatest, uint, S_IRUGO | S_IWUSR);
  61MODULE_PARM_DESC(dmatest,
  62                "dmatest 0-memcpy 1-slave_sg (default: 0)");
  63
  64static unsigned int xor_sources = 3;
  65module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  66MODULE_PARM_DESC(xor_sources,
  67                "Number of xor source buffers (default: 3)");
  68
  69static unsigned int pq_sources = 3;
  70module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  71MODULE_PARM_DESC(pq_sources,
  72                "Number of p+q source buffers (default: 3)");
  73
  74static int timeout = 3000;
  75module_param(timeout, uint, S_IRUGO | S_IWUSR);
  76MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  77                 "Pass -1 for infinite timeout");
  78
  79static bool noverify;
  80module_param(noverify, bool, S_IRUGO | S_IWUSR);
  81MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
  82
  83static bool verbose;
  84module_param(verbose, bool, S_IRUGO | S_IWUSR);
  85MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
  86
  87/**
  88 * struct dmatest_params - test parameters.
  89 * @buf_size:           size of the memcpy test buffer
  90 * @channel:            bus ID of the channel to test
  91 * @device:             bus ID of the DMA Engine to test
  92 * @threads_per_chan:   number of threads to start per channel
  93 * @max_channels:       maximum number of channels to use
  94 * @iterations:         iterations before stopping test
  95 * @xor_sources:        number of xor source buffers
  96 * @pq_sources:         number of p+q source buffers
  97 * @timeout:            transfer timeout in msec, -1 for infinite timeout
  98 */
  99struct dmatest_params {
 100        unsigned int    buf_size;
 101        char            channel[20];
 102        char            device[32];
 103        unsigned int    threads_per_chan;
 104        unsigned int    max_channels;
 105        unsigned int    iterations;
 106        unsigned int    xor_sources;
 107        unsigned int    pq_sources;
 108        int             timeout;
 109        bool            noverify;
 110};
 111
 112/**
 113 * struct dmatest_info - test information.
 114 * @params:             test parameters
 115 * @lock:               access protection to the fields of this structure
 116 */
 117static struct dmatest_info {
 118        /* Test parameters */
 119        struct dmatest_params   params;
 120
 121        /* Internal state */
 122        struct list_head        channels;
 123        unsigned int            nr_channels;
 124        struct mutex            lock;
 125        bool                    did_init;
 126} test_info = {
 127        .channels = LIST_HEAD_INIT(test_info.channels),
 128        .lock = __MUTEX_INITIALIZER(test_info.lock),
 129};
 130
 131static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 132static int dmatest_run_get(char *val, const struct kernel_param *kp);
 133static const struct kernel_param_ops run_ops = {
 134        .set = dmatest_run_set,
 135        .get = dmatest_run_get,
 136};
 137static bool dmatest_run;
 138module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
 139MODULE_PARM_DESC(run, "Run the test (default: false)");
 140
 141/* Maximum amount of mismatched bytes in buffer to print */
 142#define MAX_ERROR_COUNT         32
 143
 144/*
 145 * Initialization patterns. All bytes in the source buffer has bit 7
 146 * set, all bytes in the destination buffer has bit 7 cleared.
 147 *
 148 * Bit 6 is set for all bytes which are to be copied by the DMA
 149 * engine. Bit 5 is set for all bytes which are to be overwritten by
 150 * the DMA engine.
 151 *
 152 * The remaining bits are the inverse of a counter which increments by
 153 * one for each byte address.
 154 */
 155#define PATTERN_SRC             0x80
 156#define PATTERN_DST             0x00
 157#define PATTERN_COPY            0x40
 158#define PATTERN_OVERWRITE       0x20
 159#define PATTERN_COUNT_MASK      0x1f
 160
 161struct dmatest_thread {
 162        struct list_head        node;
 163        struct dmatest_info     *info;
 164        struct task_struct      *task;
 165        struct dma_chan         *chan;
 166        u8                      **srcs;
 167        u8                      **dsts;
 168        enum dma_transaction_type type;
 169        bool                    done;
 170};
 171
 172struct dmatest_chan {
 173        struct list_head        node;
 174        struct dma_chan         *chan;
 175        struct list_head        threads;
 176};
 177
 178static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
 179static bool wait;
 180
 181static bool is_threaded_test_run(struct dmatest_info *info)
 182{
 183        struct dmatest_chan *dtc;
 184
 185        list_for_each_entry(dtc, &info->channels, node) {
 186                struct dmatest_thread *thread;
 187
 188                list_for_each_entry(thread, &dtc->threads, node) {
 189                        if (!thread->done)
 190                                return true;
 191                }
 192        }
 193
 194        return false;
 195}
 196
 197static int dmatest_wait_get(char *val, const struct kernel_param *kp)
 198{
 199        struct dmatest_info *info = &test_info;
 200        struct dmatest_params *params = &info->params;
 201
 202        if (params->iterations)
 203                wait_event(thread_wait, !is_threaded_test_run(info));
 204        wait = true;
 205        return param_get_bool(val, kp);
 206}
 207
 208static const struct kernel_param_ops wait_ops = {
 209        .get = dmatest_wait_get,
 210        .set = param_set_bool,
 211};
 212module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
 213MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
 214
 215static bool dmatest_match_channel(struct dmatest_params *params,
 216                struct dma_chan *chan)
 217{
 218        if (params->channel[0] == '\0')
 219                return true;
 220        return strcmp(dma_chan_name(chan), params->channel) == 0;
 221}
 222
 223static bool dmatest_match_device(struct dmatest_params *params,
 224                struct dma_device *device)
 225{
 226        if (params->device[0] == '\0')
 227                return true;
 228        return strcmp(dev_name(device->dev), params->device) == 0;
 229}
 230
 231static unsigned long dmatest_random(void)
 232{
 233        unsigned long buf;
 234
 235        prandom_bytes(&buf, sizeof(buf));
 236        return buf;
 237}
 238
 239static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 240                unsigned int buf_size)
 241{
 242        unsigned int i;
 243        u8 *buf;
 244
 245        for (; (buf = *bufs); bufs++) {
 246                for (i = 0; i < start; i++)
 247                        buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 248                for ( ; i < start + len; i++)
 249                        buf[i] = PATTERN_SRC | PATTERN_COPY
 250                                | (~i & PATTERN_COUNT_MASK);
 251                for ( ; i < buf_size; i++)
 252                        buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 253                buf++;
 254        }
 255}
 256
 257static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 258                unsigned int buf_size)
 259{
 260        unsigned int i;
 261        u8 *buf;
 262
 263        for (; (buf = *bufs); bufs++) {
 264                for (i = 0; i < start; i++)
 265                        buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 266                for ( ; i < start + len; i++)
 267                        buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 268                                | (~i & PATTERN_COUNT_MASK);
 269                for ( ; i < buf_size; i++)
 270                        buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 271        }
 272}
 273
 274static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 275                unsigned int counter, bool is_srcbuf)
 276{
 277        u8              diff = actual ^ pattern;
 278        u8              expected = pattern | (~counter & PATTERN_COUNT_MASK);
 279        const char      *thread_name = current->comm;
 280
 281        if (is_srcbuf)
 282                pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 283                        thread_name, index, expected, actual);
 284        else if ((pattern & PATTERN_COPY)
 285                        && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 286                pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 287                        thread_name, index, expected, actual);
 288        else if (diff & PATTERN_SRC)
 289                pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 290                        thread_name, index, expected, actual);
 291        else
 292                pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 293                        thread_name, index, expected, actual);
 294}
 295
 296static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 297                unsigned int end, unsigned int counter, u8 pattern,
 298                bool is_srcbuf)
 299{
 300        unsigned int i;
 301        unsigned int error_count = 0;
 302        u8 actual;
 303        u8 expected;
 304        u8 *buf;
 305        unsigned int counter_orig = counter;
 306
 307        for (; (buf = *bufs); bufs++) {
 308                counter = counter_orig;
 309                for (i = start; i < end; i++) {
 310                        actual = buf[i];
 311                        expected = pattern | (~counter & PATTERN_COUNT_MASK);
 312                        if (actual != expected) {
 313                                if (error_count < MAX_ERROR_COUNT)
 314                                        dmatest_mismatch(actual, pattern, i,
 315                                                         counter, is_srcbuf);
 316                                error_count++;
 317                        }
 318                        counter++;
 319                }
 320        }
 321
 322        if (error_count > MAX_ERROR_COUNT)
 323                pr_warn("%s: %u errors suppressed\n",
 324                        current->comm, error_count - MAX_ERROR_COUNT);
 325
 326        return error_count;
 327}
 328
 329/* poor man's completion - we want to use wait_event_freezable() on it */
 330struct dmatest_done {
 331        bool                    done;
 332        wait_queue_head_t       *wait;
 333};
 334
 335static void dmatest_callback(void *arg)
 336{
 337        struct dmatest_done *done = arg;
 338
 339        done->done = true;
 340        wake_up_all(done->wait);
 341}
 342
 343static unsigned int min_odd(unsigned int x, unsigned int y)
 344{
 345        unsigned int val = min(x, y);
 346
 347        return val % 2 ? val : val - 1;
 348}
 349
 350static void result(const char *err, unsigned int n, unsigned int src_off,
 351                   unsigned int dst_off, unsigned int len, unsigned long data)
 352{
 353        pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 354                current->comm, n, err, src_off, dst_off, len, data);
 355}
 356
 357static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 358                       unsigned int dst_off, unsigned int len,
 359                       unsigned long data)
 360{
 361        pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 362                 current->comm, n, err, src_off, dst_off, len, data);
 363}
 364
 365#define verbose_result(err, n, src_off, dst_off, len, data) ({  \
 366        if (verbose)                                            \
 367                result(err, n, src_off, dst_off, len, data);    \
 368        else                                                    \
 369                dbg_result(err, n, src_off, dst_off, len, data);\
 370})
 371
 372static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 373{
 374        unsigned long long per_sec = 1000000;
 375
 376        if (runtime <= 0)
 377                return 0;
 378
 379        /* drop precision until runtime is 32-bits */
 380        while (runtime > UINT_MAX) {
 381                runtime >>= 1;
 382                per_sec <<= 1;
 383        }
 384
 385        per_sec *= val;
 386        do_div(per_sec, runtime);
 387        return per_sec;
 388}
 389
 390static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 391{
 392        return dmatest_persec(runtime, len >> 10);
 393}
 394
 395/*
 396 * This function repeatedly tests DMA transfers of various lengths and
 397 * offsets for a given operation type until it is told to exit by
 398 * kthread_stop(). There may be multiple threads running this function
 399 * in parallel for a single channel, and there may be multiple channels
 400 * being tested in parallel.
 401 *
 402 * Before each test, the source and destination buffer is initialized
 403 * with a known pattern. This pattern is different depending on
 404 * whether it's in an area which is supposed to be copied or
 405 * overwritten, and different in the source and destination buffers.
 406 * So if the DMA engine doesn't copy exactly what we tell it to copy,
 407 * we'll notice.
 408 */
 409static int dmatest_func(void *data)
 410{
 411        DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 412        struct dmatest_thread   *thread = data;
 413        struct dmatest_done     done = { .wait = &done_wait };
 414        struct dmatest_info     *info;
 415        struct dmatest_params   *params;
 416        struct dma_chan         *chan;
 417        struct dma_device       *dev;
 418        unsigned int            error_count;
 419        unsigned int            failed_tests = 0;
 420        unsigned int            total_tests = 0;
 421        dma_cookie_t            cookie;
 422        enum dma_status         status;
 423        enum dma_ctrl_flags     flags;
 424        u8                      *pq_coefs = NULL;
 425        int                     ret;
 426        int                     src_cnt;
 427        int                     dst_cnt;
 428        int                     i;
 429        ktime_t                 ktime, start, diff;
 430        ktime_t                 filltime = ktime_set(0, 0);
 431        ktime_t                 comparetime = ktime_set(0, 0);
 432        s64                     runtime = 0;
 433        unsigned long long      total_len = 0;
 434
 435        set_freezable();
 436
 437        ret = -ENOMEM;
 438
 439        smp_rmb();
 440        info = thread->info;
 441        params = &info->params;
 442        chan = thread->chan;
 443        dev = chan->device;
 444        if (thread->type == DMA_MEMCPY)
 445                src_cnt = dst_cnt = 1;
 446        else if (thread->type == DMA_SG)
 447                src_cnt = dst_cnt = sg_buffers;
 448        else if (thread->type == DMA_XOR) {
 449                /* force odd to ensure dst = src */
 450                src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 451                dst_cnt = 1;
 452        } else if (thread->type == DMA_PQ) {
 453                /* force odd to ensure dst = src */
 454                src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 455                dst_cnt = 2;
 456
 457                pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 458                if (!pq_coefs)
 459                        goto err_thread_type;
 460
 461                for (i = 0; i < src_cnt; i++)
 462                        pq_coefs[i] = 1;
 463        } else
 464                goto err_thread_type;
 465
 466        thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 467        if (!thread->srcs)
 468                goto err_srcs;
 469        for (i = 0; i < src_cnt; i++) {
 470                thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 471                if (!thread->srcs[i])
 472                        goto err_srcbuf;
 473        }
 474        thread->srcs[i] = NULL;
 475
 476        thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 477        if (!thread->dsts)
 478                goto err_dsts;
 479        for (i = 0; i < dst_cnt; i++) {
 480                thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 481                if (!thread->dsts[i])
 482                        goto err_dstbuf;
 483        }
 484        thread->dsts[i] = NULL;
 485
 486        set_user_nice(current, 10);
 487
 488        /*
 489         * src and dst buffers are freed by ourselves below
 490         */
 491        flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 492
 493        ktime = ktime_get();
 494        while (!kthread_should_stop()
 495               && !(params->iterations && total_tests >= params->iterations)) {
 496                struct dma_async_tx_descriptor *tx = NULL;
 497                struct dmaengine_unmap_data *um;
 498                dma_addr_t srcs[src_cnt];
 499                dma_addr_t *dsts;
 500                unsigned int src_off, dst_off, len;
 501                u8 align = 0;
 502                struct scatterlist tx_sg[src_cnt];
 503                struct scatterlist rx_sg[src_cnt];
 504
 505                total_tests++;
 506
 507                /* honor alignment restrictions */
 508                if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
 509                        align = dev->copy_align;
 510                else if (thread->type == DMA_XOR)
 511                        align = dev->xor_align;
 512                else if (thread->type == DMA_PQ)
 513                        align = dev->pq_align;
 514
 515                if (1 << align > params->buf_size) {
 516                        pr_err("%u-byte buffer too small for %d-byte alignment\n",
 517                               params->buf_size, 1 << align);
 518                        break;
 519                }
 520
 521                align = 3;
 522
 523                if (params->noverify)
 524                        len = params->buf_size;
 525                else
 526                        len = dmatest_random() % params->buf_size + 1;
 527
 528                len = (len >> align) << align;
 529                if (!len)
 530                        len = 1 << align;
 531
 532                total_len += len;
 533
 534                if (params->noverify) {
 535                        src_off = 0;
 536                        dst_off = 0;
 537                } else {
 538                        start = ktime_get();
 539                        src_off = dmatest_random() % (params->buf_size - len + 1);
 540                        dst_off = dmatest_random() % (params->buf_size - len + 1);
 541
 542                        src_off = (src_off >> align) << align;
 543                        dst_off = (dst_off >> align) << align;
 544
 545                        dmatest_init_srcs(thread->srcs, src_off, len,
 546                                          params->buf_size);
 547                        dmatest_init_dsts(thread->dsts, dst_off, len,
 548                                          params->buf_size);
 549
 550                        diff = ktime_sub(ktime_get(), start);
 551                        filltime = ktime_add(filltime, diff);
 552                }
 553
 554                um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
 555                                              GFP_KERNEL);
 556                if (!um) {
 557                        failed_tests++;
 558                        result("unmap data NULL", total_tests,
 559                               src_off, dst_off, len, ret);
 560                        continue;
 561                }
 562
 563                um->len = params->buf_size;
 564                for (i = 0; i < src_cnt; i++) {
 565                        void *buf = thread->srcs[i];
 566                        struct page *pg = virt_to_page(buf);
 567                        unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
 568
 569                        um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
 570                                                   um->len, DMA_TO_DEVICE);
 571                        srcs[i] = um->addr[i] + src_off;
 572                        ret = dma_mapping_error(dev->dev, um->addr[i]);
 573                        if (ret) {
 574                                dmaengine_unmap_put(um);
 575                                result("src mapping error", total_tests,
 576                                       src_off, dst_off, len, ret);
 577                                failed_tests++;
 578                                continue;
 579                        }
 580                        um->to_cnt++;
 581                }
 582                /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 583                dsts = &um->addr[src_cnt];
 584                for (i = 0; i < dst_cnt; i++) {
 585                        void *buf = thread->dsts[i];
 586                        struct page *pg = virt_to_page(buf);
 587                        unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
 588
 589                        dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
 590                                               DMA_BIDIRECTIONAL);
 591                        ret = dma_mapping_error(dev->dev, dsts[i]);
 592                        if (ret) {
 593                                dmaengine_unmap_put(um);
 594                                result("dst mapping error", total_tests,
 595                                       src_off, dst_off, len, ret);
 596                                failed_tests++;
 597                                continue;
 598                        }
 599                        um->bidi_cnt++;
 600                }
 601
 602                sg_init_table(tx_sg, src_cnt);
 603                sg_init_table(rx_sg, src_cnt);
 604                for (i = 0; i < src_cnt; i++) {
 605                        sg_dma_address(&rx_sg[i]) = srcs[i];
 606                        sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
 607                        sg_dma_len(&tx_sg[i]) = len;
 608                        sg_dma_len(&rx_sg[i]) = len;
 609                }
 610
 611                if (thread->type == DMA_MEMCPY)
 612                        tx = dev->device_prep_dma_memcpy(chan,
 613                                                         dsts[0] + dst_off,
 614                                                         srcs[0], len, flags);
 615                else if (thread->type == DMA_SG)
 616                        tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
 617                                                     rx_sg, src_cnt, flags);
 618                else if (thread->type == DMA_XOR)
 619                        tx = dev->device_prep_dma_xor(chan,
 620                                                      dsts[0] + dst_off,
 621                                                      srcs, src_cnt,
 622                                                      len, flags);
 623                else if (thread->type == DMA_PQ) {
 624                        dma_addr_t dma_pq[dst_cnt];
 625
 626                        for (i = 0; i < dst_cnt; i++)
 627                                dma_pq[i] = dsts[i] + dst_off;
 628                        tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
 629                                                     src_cnt, pq_coefs,
 630                                                     len, flags);
 631                }
 632
 633                if (!tx) {
 634                        dmaengine_unmap_put(um);
 635                        result("prep error", total_tests, src_off,
 636                               dst_off, len, ret);
 637                        msleep(100);
 638                        failed_tests++;
 639                        continue;
 640                }
 641
 642                done.done = false;
 643                tx->callback = dmatest_callback;
 644                tx->callback_param = &done;
 645                cookie = tx->tx_submit(tx);
 646
 647                if (dma_submit_error(cookie)) {
 648                        dmaengine_unmap_put(um);
 649                        result("submit error", total_tests, src_off,
 650                               dst_off, len, ret);
 651                        msleep(100);
 652                        failed_tests++;
 653                        continue;
 654                }
 655                dma_async_issue_pending(chan);
 656
 657                wait_event_freezable_timeout(done_wait, done.done,
 658                                             msecs_to_jiffies(params->timeout));
 659
 660                status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 661
 662                if (!done.done) {
 663                        /*
 664                         * We're leaving the timed out dma operation with
 665                         * dangling pointer to done_wait.  To make this
 666                         * correct, we'll need to allocate wait_done for
 667                         * each test iteration and perform "who's gonna
 668                         * free it this time?" dancing.  For now, just
 669                         * leave it dangling.
 670                         */
 671                        dmaengine_unmap_put(um);
 672                        result("test timed out", total_tests, src_off, dst_off,
 673                               len, 0);
 674                        failed_tests++;
 675                        continue;
 676                } else if (status != DMA_COMPLETE) {
 677                        dmaengine_unmap_put(um);
 678                        result(status == DMA_ERROR ?
 679                               "completion error status" :
 680                               "completion busy status", total_tests, src_off,
 681                               dst_off, len, ret);
 682                        failed_tests++;
 683                        continue;
 684                }
 685
 686                dmaengine_unmap_put(um);
 687
 688                if (params->noverify) {
 689                        verbose_result("test passed", total_tests, src_off,
 690                                       dst_off, len, 0);
 691                        continue;
 692                }
 693
 694                start = ktime_get();
 695                pr_debug("%s: verifying source buffer...\n", current->comm);
 696                error_count = dmatest_verify(thread->srcs, 0, src_off,
 697                                0, PATTERN_SRC, true);
 698                error_count += dmatest_verify(thread->srcs, src_off,
 699                                src_off + len, src_off,
 700                                PATTERN_SRC | PATTERN_COPY, true);
 701                error_count += dmatest_verify(thread->srcs, src_off + len,
 702                                params->buf_size, src_off + len,
 703                                PATTERN_SRC, true);
 704
 705                pr_debug("%s: verifying dest buffer...\n", current->comm);
 706                error_count += dmatest_verify(thread->dsts, 0, dst_off,
 707                                0, PATTERN_DST, false);
 708                error_count += dmatest_verify(thread->dsts, dst_off,
 709                                dst_off + len, src_off,
 710                                PATTERN_SRC | PATTERN_COPY, false);
 711                error_count += dmatest_verify(thread->dsts, dst_off + len,
 712                                params->buf_size, dst_off + len,
 713                                PATTERN_DST, false);
 714
 715                diff = ktime_sub(ktime_get(), start);
 716                comparetime = ktime_add(comparetime, diff);
 717
 718                if (error_count) {
 719                        result("data error", total_tests, src_off, dst_off,
 720                               len, error_count);
 721                        failed_tests++;
 722                } else {
 723                        verbose_result("test passed", total_tests, src_off,
 724                                       dst_off, len, 0);
 725                }
 726        }
 727        ktime = ktime_sub(ktime_get(), ktime);
 728        ktime = ktime_sub(ktime, comparetime);
 729        ktime = ktime_sub(ktime, filltime);
 730        runtime = ktime_to_us(ktime);
 731
 732        ret = 0;
 733err_dstbuf:
 734        for (i = 0; thread->dsts[i]; i++)
 735                kfree(thread->dsts[i]);
 736        kfree(thread->dsts);
 737err_dsts:
 738err_srcbuf:
 739        for (i = 0; thread->srcs[i]; i++)
 740                kfree(thread->srcs[i]);
 741        kfree(thread->srcs);
 742err_srcs:
 743        kfree(pq_coefs);
 744err_thread_type:
 745        pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
 746                current->comm, total_tests, failed_tests,
 747                dmatest_persec(runtime, total_tests),
 748                dmatest_KBs(runtime, total_len), ret);
 749
 750        /* terminate all transfers on specified channels */
 751        if (ret)
 752                dmaengine_terminate_all(chan);
 753
 754        thread->done = true;
 755        wake_up(&thread_wait);
 756
 757        return ret;
 758}
 759
 760static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 761{
 762        struct dmatest_thread   *thread;
 763        struct dmatest_thread   *_thread;
 764        int                     ret;
 765
 766        list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 767                ret = kthread_stop(thread->task);
 768                pr_debug("thread %s exited with status %d\n",
 769                         thread->task->comm, ret);
 770                list_del(&thread->node);
 771                put_task_struct(thread->task);
 772                kfree(thread);
 773        }
 774
 775        /* terminate all transfers on specified channels */
 776        dmaengine_terminate_all(dtc->chan);
 777
 778        kfree(dtc);
 779}
 780
 781static int dmatest_add_threads(struct dmatest_info *info,
 782                struct dmatest_chan *dtc, enum dma_transaction_type type)
 783{
 784        struct dmatest_params *params = &info->params;
 785        struct dmatest_thread *thread;
 786        struct dma_chan *chan = dtc->chan;
 787        char *op;
 788        unsigned int i;
 789
 790        if (type == DMA_MEMCPY)
 791                op = "copy";
 792        else if (type == DMA_SG)
 793                op = "sg";
 794        else if (type == DMA_XOR)
 795                op = "xor";
 796        else if (type == DMA_PQ)
 797                op = "pq";
 798        else
 799                return -EINVAL;
 800
 801        for (i = 0; i < params->threads_per_chan; i++) {
 802                thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 803                if (!thread) {
 804                        pr_warn("No memory for %s-%s%u\n",
 805                                dma_chan_name(chan), op, i);
 806                        break;
 807                }
 808                thread->info = info;
 809                thread->chan = dtc->chan;
 810                thread->type = type;
 811                smp_wmb();
 812                thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
 813                                dma_chan_name(chan), op, i);
 814                if (IS_ERR(thread->task)) {
 815                        pr_warn("Failed to create thread %s-%s%u\n",
 816                                dma_chan_name(chan), op, i);
 817                        kfree(thread);
 818                        break;
 819                }
 820
 821                /* srcbuf and dstbuf are allocated by the thread itself */
 822                get_task_struct(thread->task);
 823                list_add_tail(&thread->node, &dtc->threads);
 824                wake_up_process(thread->task);
 825        }
 826
 827        return i;
 828}
 829
 830static int dmatest_add_channel(struct dmatest_info *info,
 831                struct dma_chan *chan)
 832{
 833        struct dmatest_chan     *dtc;
 834        struct dma_device       *dma_dev = chan->device;
 835        unsigned int            thread_count = 0;
 836        int cnt;
 837
 838        dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 839        if (!dtc) {
 840                pr_warn("No memory for %s\n", dma_chan_name(chan));
 841                return -ENOMEM;
 842        }
 843
 844        dtc->chan = chan;
 845        INIT_LIST_HEAD(&dtc->threads);
 846
 847        if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 848                if (dmatest == 0) {
 849                        cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 850                        thread_count += cnt > 0 ? cnt : 0;
 851                }
 852        }
 853
 854        if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
 855                if (dmatest == 1) {
 856                        cnt = dmatest_add_threads(info, dtc, DMA_SG);
 857                        thread_count += cnt > 0 ? cnt : 0;
 858                }
 859        }
 860
 861        if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 862                cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 863                thread_count += cnt > 0 ? cnt : 0;
 864        }
 865        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 866                cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 867                thread_count += cnt > 0 ? cnt : 0;
 868        }
 869
 870        pr_info("Started %u threads using %s\n",
 871                thread_count, dma_chan_name(chan));
 872
 873        list_add_tail(&dtc->node, &info->channels);
 874        info->nr_channels++;
 875
 876        return 0;
 877}
 878
 879static bool filter(struct dma_chan *chan, void *param)
 880{
 881        struct dmatest_params *params = param;
 882
 883        if (!dmatest_match_channel(params, chan) ||
 884            !dmatest_match_device(params, chan->device))
 885                return false;
 886        else
 887                return true;
 888}
 889
 890static void request_channels(struct dmatest_info *info,
 891                             enum dma_transaction_type type)
 892{
 893        dma_cap_mask_t mask;
 894
 895        dma_cap_zero(mask);
 896        dma_cap_set(type, mask);
 897        for (;;) {
 898                struct dmatest_params *params = &info->params;
 899                struct dma_chan *chan;
 900
 901                chan = dma_request_channel(mask, filter, params);
 902                if (chan) {
 903                        if (dmatest_add_channel(info, chan)) {
 904                                dma_release_channel(chan);
 905                                break; /* add_channel failed, punt */
 906                        }
 907                } else
 908                        break; /* no more channels available */
 909                if (params->max_channels &&
 910                    info->nr_channels >= params->max_channels)
 911                        break; /* we have all we need */
 912        }
 913}
 914
 915static void run_threaded_test(struct dmatest_info *info)
 916{
 917        struct dmatest_params *params = &info->params;
 918
 919        /* Copy test parameters */
 920        params->buf_size = test_buf_size;
 921        strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
 922        strlcpy(params->device, strim(test_device), sizeof(params->device));
 923        params->threads_per_chan = threads_per_chan;
 924        params->max_channels = max_channels;
 925        params->iterations = iterations;
 926        params->xor_sources = xor_sources;
 927        params->pq_sources = pq_sources;
 928        params->timeout = timeout;
 929        params->noverify = noverify;
 930
 931        request_channels(info, DMA_MEMCPY);
 932        request_channels(info, DMA_XOR);
 933        request_channels(info, DMA_SG);
 934        request_channels(info, DMA_PQ);
 935}
 936
 937static void stop_threaded_test(struct dmatest_info *info)
 938{
 939        struct dmatest_chan *dtc, *_dtc;
 940        struct dma_chan *chan;
 941
 942        list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 943                list_del(&dtc->node);
 944                chan = dtc->chan;
 945                dmatest_cleanup_channel(dtc);
 946                pr_debug("dropped channel %s\n", dma_chan_name(chan));
 947                dma_release_channel(chan);
 948        }
 949
 950        info->nr_channels = 0;
 951}
 952
 953static void restart_threaded_test(struct dmatest_info *info, bool run)
 954{
 955        /* we might be called early to set run=, defer running until all
 956         * parameters have been evaluated
 957         */
 958        if (!info->did_init)
 959                return;
 960
 961        /* Stop any running test first */
 962        stop_threaded_test(info);
 963
 964        /* Run test with new parameters */
 965        run_threaded_test(info);
 966}
 967
 968static int dmatest_run_get(char *val, const struct kernel_param *kp)
 969{
 970        struct dmatest_info *info = &test_info;
 971
 972        mutex_lock(&info->lock);
 973        if (is_threaded_test_run(info)) {
 974                dmatest_run = true;
 975        } else {
 976                stop_threaded_test(info);
 977                dmatest_run = false;
 978        }
 979        mutex_unlock(&info->lock);
 980
 981        return param_get_bool(val, kp);
 982}
 983
 984static int dmatest_run_set(const char *val, const struct kernel_param *kp)
 985{
 986        struct dmatest_info *info = &test_info;
 987        int ret;
 988
 989        mutex_lock(&info->lock);
 990        ret = param_set_bool(val, kp);
 991        if (ret) {
 992                mutex_unlock(&info->lock);
 993                return ret;
 994        }
 995
 996        if (is_threaded_test_run(info))
 997                ret = -EBUSY;
 998        else if (dmatest_run)
 999                restart_threaded_test(info, dmatest_run);
1000
1001        mutex_unlock(&info->lock);
1002
1003        return ret;
1004}
1005
1006static int __init dmatest_init(void)
1007{
1008        struct dmatest_info *info = &test_info;
1009        struct dmatest_params *params = &info->params;
1010
1011        if (dmatest_run) {
1012                mutex_lock(&info->lock);
1013                run_threaded_test(info);
1014                mutex_unlock(&info->lock);
1015        }
1016
1017        if (params->iterations && wait)
1018                wait_event(thread_wait, !is_threaded_test_run(info));
1019
1020        /* module parameters are stable, inittime tests are started,
1021         * let userspace take over 'run' control
1022         */
1023        info->did_init = true;
1024
1025        return 0;
1026}
1027/* when compiled-in wait for drivers to load first */
1028late_initcall(dmatest_init);
1029
1030static void __exit dmatest_exit(void)
1031{
1032        struct dmatest_info *info = &test_info;
1033
1034        mutex_lock(&info->lock);
1035        stop_threaded_test(info);
1036        mutex_unlock(&info->lock);
1037}
1038module_exit(dmatest_exit);
1039
1040MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1041MODULE_LICENSE("GPL v2");
1042