linux/drivers/dma/imx-dma.c
<<
>>
Prefs
   1/*
   2 * drivers/dma/imx-dma.c
   3 *
   4 * This file contains a driver for the Freescale i.MX DMA engine
   5 * found on i.MX1/21/27
   6 *
   7 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
   8 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
   9 *
  10 * The code contained herein is licensed under the GNU General Public
  11 * License. You may obtain a copy of the GNU General Public License
  12 * Version 2 or later at the following locations:
  13 *
  14 * http://www.opensource.org/licenses/gpl-license.html
  15 * http://www.gnu.org/copyleft/gpl.html
  16 */
  17#include <linux/err.h>
  18#include <linux/init.h>
  19#include <linux/types.h>
  20#include <linux/mm.h>
  21#include <linux/interrupt.h>
  22#include <linux/spinlock.h>
  23#include <linux/device.h>
  24#include <linux/dma-mapping.h>
  25#include <linux/slab.h>
  26#include <linux/platform_device.h>
  27#include <linux/clk.h>
  28#include <linux/dmaengine.h>
  29#include <linux/module.h>
  30#include <linux/of_device.h>
  31#include <linux/of_dma.h>
  32
  33#include <asm/irq.h>
  34#include <linux/platform_data/dma-imx.h>
  35
  36#include "dmaengine.h"
  37#define IMXDMA_MAX_CHAN_DESCRIPTORS     16
  38#define IMX_DMA_CHANNELS  16
  39
  40#define IMX_DMA_2D_SLOTS        2
  41#define IMX_DMA_2D_SLOT_A       0
  42#define IMX_DMA_2D_SLOT_B       1
  43
  44#define IMX_DMA_LENGTH_LOOP     ((unsigned int)-1)
  45#define IMX_DMA_MEMSIZE_32      (0 << 4)
  46#define IMX_DMA_MEMSIZE_8       (1 << 4)
  47#define IMX_DMA_MEMSIZE_16      (2 << 4)
  48#define IMX_DMA_TYPE_LINEAR     (0 << 10)
  49#define IMX_DMA_TYPE_2D         (1 << 10)
  50#define IMX_DMA_TYPE_FIFO       (2 << 10)
  51
  52#define IMX_DMA_ERR_BURST     (1 << 0)
  53#define IMX_DMA_ERR_REQUEST   (1 << 1)
  54#define IMX_DMA_ERR_TRANSFER  (1 << 2)
  55#define IMX_DMA_ERR_BUFFER    (1 << 3)
  56#define IMX_DMA_ERR_TIMEOUT   (1 << 4)
  57
  58#define DMA_DCR     0x00                /* Control Register */
  59#define DMA_DISR    0x04                /* Interrupt status Register */
  60#define DMA_DIMR    0x08                /* Interrupt mask Register */
  61#define DMA_DBTOSR  0x0c                /* Burst timeout status Register */
  62#define DMA_DRTOSR  0x10                /* Request timeout Register */
  63#define DMA_DSESR   0x14                /* Transfer Error Status Register */
  64#define DMA_DBOSR   0x18                /* Buffer overflow status Register */
  65#define DMA_DBTOCR  0x1c                /* Burst timeout control Register */
  66#define DMA_WSRA    0x40                /* W-Size Register A */
  67#define DMA_XSRA    0x44                /* X-Size Register A */
  68#define DMA_YSRA    0x48                /* Y-Size Register A */
  69#define DMA_WSRB    0x4c                /* W-Size Register B */
  70#define DMA_XSRB    0x50                /* X-Size Register B */
  71#define DMA_YSRB    0x54                /* Y-Size Register B */
  72#define DMA_SAR(x)  (0x80 + ((x) << 6)) /* Source Address Registers */
  73#define DMA_DAR(x)  (0x84 + ((x) << 6)) /* Destination Address Registers */
  74#define DMA_CNTR(x) (0x88 + ((x) << 6)) /* Count Registers */
  75#define DMA_CCR(x)  (0x8c + ((x) << 6)) /* Control Registers */
  76#define DMA_RSSR(x) (0x90 + ((x) << 6)) /* Request source select Registers */
  77#define DMA_BLR(x)  (0x94 + ((x) << 6)) /* Burst length Registers */
  78#define DMA_RTOR(x) (0x98 + ((x) << 6)) /* Request timeout Registers */
  79#define DMA_BUCR(x) (0x98 + ((x) << 6)) /* Bus Utilization Registers */
  80#define DMA_CCNR(x) (0x9C + ((x) << 6)) /* Channel counter Registers */
  81
  82#define DCR_DRST           (1<<1)
  83#define DCR_DEN            (1<<0)
  84#define DBTOCR_EN          (1<<15)
  85#define DBTOCR_CNT(x)      ((x) & 0x7fff)
  86#define CNTR_CNT(x)        ((x) & 0xffffff)
  87#define CCR_ACRPT          (1<<14)
  88#define CCR_DMOD_LINEAR    (0x0 << 12)
  89#define CCR_DMOD_2D        (0x1 << 12)
  90#define CCR_DMOD_FIFO      (0x2 << 12)
  91#define CCR_DMOD_EOBFIFO   (0x3 << 12)
  92#define CCR_SMOD_LINEAR    (0x0 << 10)
  93#define CCR_SMOD_2D        (0x1 << 10)
  94#define CCR_SMOD_FIFO      (0x2 << 10)
  95#define CCR_SMOD_EOBFIFO   (0x3 << 10)
  96#define CCR_MDIR_DEC       (1<<9)
  97#define CCR_MSEL_B         (1<<8)
  98#define CCR_DSIZ_32        (0x0 << 6)
  99#define CCR_DSIZ_8         (0x1 << 6)
 100#define CCR_DSIZ_16        (0x2 << 6)
 101#define CCR_SSIZ_32        (0x0 << 4)
 102#define CCR_SSIZ_8         (0x1 << 4)
 103#define CCR_SSIZ_16        (0x2 << 4)
 104#define CCR_REN            (1<<3)
 105#define CCR_RPT            (1<<2)
 106#define CCR_FRC            (1<<1)
 107#define CCR_CEN            (1<<0)
 108#define RTOR_EN            (1<<15)
 109#define RTOR_CLK           (1<<14)
 110#define RTOR_PSC           (1<<13)
 111
 112enum  imxdma_prep_type {
 113        IMXDMA_DESC_MEMCPY,
 114        IMXDMA_DESC_INTERLEAVED,
 115        IMXDMA_DESC_SLAVE_SG,
 116        IMXDMA_DESC_CYCLIC,
 117};
 118
 119struct imx_dma_2d_config {
 120        u16             xsr;
 121        u16             ysr;
 122        u16             wsr;
 123        int             count;
 124};
 125
 126struct imxdma_desc {
 127        struct list_head                node;
 128        struct dma_async_tx_descriptor  desc;
 129        enum dma_status                 status;
 130        dma_addr_t                      src;
 131        dma_addr_t                      dest;
 132        size_t                          len;
 133        enum dma_transfer_direction     direction;
 134        enum imxdma_prep_type           type;
 135        /* For memcpy and interleaved */
 136        unsigned int                    config_port;
 137        unsigned int                    config_mem;
 138        /* For interleaved transfers */
 139        unsigned int                    x;
 140        unsigned int                    y;
 141        unsigned int                    w;
 142        /* For slave sg and cyclic */
 143        struct scatterlist              *sg;
 144        unsigned int                    sgcount;
 145};
 146
 147struct imxdma_channel {
 148        int                             hw_chaining;
 149        struct timer_list               watchdog;
 150        struct imxdma_engine            *imxdma;
 151        unsigned int                    channel;
 152
 153        struct tasklet_struct           dma_tasklet;
 154        struct list_head                ld_free;
 155        struct list_head                ld_queue;
 156        struct list_head                ld_active;
 157        int                             descs_allocated;
 158        enum dma_slave_buswidth         word_size;
 159        dma_addr_t                      per_address;
 160        u32                             watermark_level;
 161        struct dma_chan                 chan;
 162        struct dma_async_tx_descriptor  desc;
 163        enum dma_status                 status;
 164        int                             dma_request;
 165        struct scatterlist              *sg_list;
 166        u32                             ccr_from_device;
 167        u32                             ccr_to_device;
 168        bool                            enabled_2d;
 169        int                             slot_2d;
 170};
 171
 172enum imx_dma_type {
 173        IMX1_DMA,
 174        IMX21_DMA,
 175        IMX27_DMA,
 176};
 177
 178struct imxdma_engine {
 179        struct device                   *dev;
 180        struct device_dma_parameters    dma_parms;
 181        struct dma_device               dma_device;
 182        void __iomem                    *base;
 183        struct clk                      *dma_ahb;
 184        struct clk                      *dma_ipg;
 185        spinlock_t                      lock;
 186        struct imx_dma_2d_config        slots_2d[IMX_DMA_2D_SLOTS];
 187        struct imxdma_channel           channel[IMX_DMA_CHANNELS];
 188        enum imx_dma_type               devtype;
 189};
 190
 191struct imxdma_filter_data {
 192        struct imxdma_engine    *imxdma;
 193        int                      request;
 194};
 195
 196static struct platform_device_id imx_dma_devtype[] = {
 197        {
 198                .name = "imx1-dma",
 199                .driver_data = IMX1_DMA,
 200        }, {
 201                .name = "imx21-dma",
 202                .driver_data = IMX21_DMA,
 203        }, {
 204                .name = "imx27-dma",
 205                .driver_data = IMX27_DMA,
 206        }, {
 207                /* sentinel */
 208        }
 209};
 210MODULE_DEVICE_TABLE(platform, imx_dma_devtype);
 211
 212static const struct of_device_id imx_dma_of_dev_id[] = {
 213        {
 214                .compatible = "fsl,imx1-dma",
 215                .data = &imx_dma_devtype[IMX1_DMA],
 216        }, {
 217                .compatible = "fsl,imx21-dma",
 218                .data = &imx_dma_devtype[IMX21_DMA],
 219        }, {
 220                .compatible = "fsl,imx27-dma",
 221                .data = &imx_dma_devtype[IMX27_DMA],
 222        }, {
 223                /* sentinel */
 224        }
 225};
 226MODULE_DEVICE_TABLE(of, imx_dma_of_dev_id);
 227
 228static inline int is_imx1_dma(struct imxdma_engine *imxdma)
 229{
 230        return imxdma->devtype == IMX1_DMA;
 231}
 232
 233static inline int is_imx27_dma(struct imxdma_engine *imxdma)
 234{
 235        return imxdma->devtype == IMX27_DMA;
 236}
 237
 238static struct imxdma_channel *to_imxdma_chan(struct dma_chan *chan)
 239{
 240        return container_of(chan, struct imxdma_channel, chan);
 241}
 242
 243static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)
 244{
 245        struct imxdma_desc *desc;
 246
 247        if (!list_empty(&imxdmac->ld_active)) {
 248                desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,
 249                                        node);
 250                if (desc->type == IMXDMA_DESC_CYCLIC)
 251                        return true;
 252        }
 253        return false;
 254}
 255
 256
 257
 258static void imx_dmav1_writel(struct imxdma_engine *imxdma, unsigned val,
 259                             unsigned offset)
 260{
 261        __raw_writel(val, imxdma->base + offset);
 262}
 263
 264static unsigned imx_dmav1_readl(struct imxdma_engine *imxdma, unsigned offset)
 265{
 266        return __raw_readl(imxdma->base + offset);
 267}
 268
 269static int imxdma_hw_chain(struct imxdma_channel *imxdmac)
 270{
 271        struct imxdma_engine *imxdma = imxdmac->imxdma;
 272
 273        if (is_imx27_dma(imxdma))
 274                return imxdmac->hw_chaining;
 275        else
 276                return 0;
 277}
 278
 279/*
 280 * imxdma_sg_next - prepare next chunk for scatter-gather DMA emulation
 281 */
 282static inline int imxdma_sg_next(struct imxdma_desc *d)
 283{
 284        struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
 285        struct imxdma_engine *imxdma = imxdmac->imxdma;
 286        struct scatterlist *sg = d->sg;
 287        unsigned long now;
 288
 289        now = min(d->len, sg_dma_len(sg));
 290        if (d->len != IMX_DMA_LENGTH_LOOP)
 291                d->len -= now;
 292
 293        if (d->direction == DMA_DEV_TO_MEM)
 294                imx_dmav1_writel(imxdma, sg->dma_address,
 295                                 DMA_DAR(imxdmac->channel));
 296        else
 297                imx_dmav1_writel(imxdma, sg->dma_address,
 298                                 DMA_SAR(imxdmac->channel));
 299
 300        imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));
 301
 302        dev_dbg(imxdma->dev, " %s channel: %d dst 0x%08x, src 0x%08x, "
 303                "size 0x%08x\n", __func__, imxdmac->channel,
 304                 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),
 305                 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),
 306                 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));
 307
 308        return now;
 309}
 310
 311static void imxdma_enable_hw(struct imxdma_desc *d)
 312{
 313        struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
 314        struct imxdma_engine *imxdma = imxdmac->imxdma;
 315        int channel = imxdmac->channel;
 316        unsigned long flags;
 317
 318        dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);
 319
 320        local_irq_save(flags);
 321
 322        imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);
 323        imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) &
 324                         ~(1 << channel), DMA_DIMR);
 325        imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) |
 326                         CCR_CEN | CCR_ACRPT, DMA_CCR(channel));
 327
 328        if (!is_imx1_dma(imxdma) &&
 329                        d->sg && imxdma_hw_chain(imxdmac)) {
 330                d->sg = sg_next(d->sg);
 331                if (d->sg) {
 332                        u32 tmp;
 333                        imxdma_sg_next(d);
 334                        tmp = imx_dmav1_readl(imxdma, DMA_CCR(channel));
 335                        imx_dmav1_writel(imxdma, tmp | CCR_RPT | CCR_ACRPT,
 336                                         DMA_CCR(channel));
 337                }
 338        }
 339
 340        local_irq_restore(flags);
 341}
 342
 343static void imxdma_disable_hw(struct imxdma_channel *imxdmac)
 344{
 345        struct imxdma_engine *imxdma = imxdmac->imxdma;
 346        int channel = imxdmac->channel;
 347        unsigned long flags;
 348
 349        dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);
 350
 351        if (imxdma_hw_chain(imxdmac))
 352                del_timer(&imxdmac->watchdog);
 353
 354        local_irq_save(flags);
 355        imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) |
 356                         (1 << channel), DMA_DIMR);
 357        imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) &
 358                         ~CCR_CEN, DMA_CCR(channel));
 359        imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);
 360        local_irq_restore(flags);
 361}
 362
 363static void imxdma_watchdog(unsigned long data)
 364{
 365        struct imxdma_channel *imxdmac = (struct imxdma_channel *)data;
 366        struct imxdma_engine *imxdma = imxdmac->imxdma;
 367        int channel = imxdmac->channel;
 368
 369        imx_dmav1_writel(imxdma, 0, DMA_CCR(channel));
 370
 371        /* Tasklet watchdog error handler */
 372        tasklet_schedule(&imxdmac->dma_tasklet);
 373        dev_dbg(imxdma->dev, "channel %d: watchdog timeout!\n",
 374                imxdmac->channel);
 375}
 376
 377static irqreturn_t imxdma_err_handler(int irq, void *dev_id)
 378{
 379        struct imxdma_engine *imxdma = dev_id;
 380        unsigned int err_mask;
 381        int i, disr;
 382        int errcode;
 383
 384        disr = imx_dmav1_readl(imxdma, DMA_DISR);
 385
 386        err_mask = imx_dmav1_readl(imxdma, DMA_DBTOSR) |
 387                   imx_dmav1_readl(imxdma, DMA_DRTOSR) |
 388                   imx_dmav1_readl(imxdma, DMA_DSESR)  |
 389                   imx_dmav1_readl(imxdma, DMA_DBOSR);
 390
 391        if (!err_mask)
 392                return IRQ_HANDLED;
 393
 394        imx_dmav1_writel(imxdma, disr & err_mask, DMA_DISR);
 395
 396        for (i = 0; i < IMX_DMA_CHANNELS; i++) {
 397                if (!(err_mask & (1 << i)))
 398                        continue;
 399                errcode = 0;
 400
 401                if (imx_dmav1_readl(imxdma, DMA_DBTOSR) & (1 << i)) {
 402                        imx_dmav1_writel(imxdma, 1 << i, DMA_DBTOSR);
 403                        errcode |= IMX_DMA_ERR_BURST;
 404                }
 405                if (imx_dmav1_readl(imxdma, DMA_DRTOSR) & (1 << i)) {
 406                        imx_dmav1_writel(imxdma, 1 << i, DMA_DRTOSR);
 407                        errcode |= IMX_DMA_ERR_REQUEST;
 408                }
 409                if (imx_dmav1_readl(imxdma, DMA_DSESR) & (1 << i)) {
 410                        imx_dmav1_writel(imxdma, 1 << i, DMA_DSESR);
 411                        errcode |= IMX_DMA_ERR_TRANSFER;
 412                }
 413                if (imx_dmav1_readl(imxdma, DMA_DBOSR) & (1 << i)) {
 414                        imx_dmav1_writel(imxdma, 1 << i, DMA_DBOSR);
 415                        errcode |= IMX_DMA_ERR_BUFFER;
 416                }
 417                /* Tasklet error handler */
 418                tasklet_schedule(&imxdma->channel[i].dma_tasklet);
 419
 420                dev_warn(imxdma->dev,
 421                         "DMA timeout on channel %d -%s%s%s%s\n", i,
 422                         errcode & IMX_DMA_ERR_BURST ?    " burst" : "",
 423                         errcode & IMX_DMA_ERR_REQUEST ?  " request" : "",
 424                         errcode & IMX_DMA_ERR_TRANSFER ? " transfer" : "",
 425                         errcode & IMX_DMA_ERR_BUFFER ?   " buffer" : "");
 426        }
 427        return IRQ_HANDLED;
 428}
 429
 430static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)
 431{
 432        struct imxdma_engine *imxdma = imxdmac->imxdma;
 433        int chno = imxdmac->channel;
 434        struct imxdma_desc *desc;
 435        unsigned long flags;
 436
 437        spin_lock_irqsave(&imxdma->lock, flags);
 438        if (list_empty(&imxdmac->ld_active)) {
 439                spin_unlock_irqrestore(&imxdma->lock, flags);
 440                goto out;
 441        }
 442
 443        desc = list_first_entry(&imxdmac->ld_active,
 444                                struct imxdma_desc,
 445                                node);
 446        spin_unlock_irqrestore(&imxdma->lock, flags);
 447
 448        if (desc->sg) {
 449                u32 tmp;
 450                desc->sg = sg_next(desc->sg);
 451
 452                if (desc->sg) {
 453                        imxdma_sg_next(desc);
 454
 455                        tmp = imx_dmav1_readl(imxdma, DMA_CCR(chno));
 456
 457                        if (imxdma_hw_chain(imxdmac)) {
 458                                /* FIXME: The timeout should probably be
 459                                 * configurable
 460                                 */
 461                                mod_timer(&imxdmac->watchdog,
 462                                        jiffies + msecs_to_jiffies(500));
 463
 464                                tmp |= CCR_CEN | CCR_RPT | CCR_ACRPT;
 465                                imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));
 466                        } else {
 467                                imx_dmav1_writel(imxdma, tmp & ~CCR_CEN,
 468                                                 DMA_CCR(chno));
 469                                tmp |= CCR_CEN;
 470                        }
 471
 472                        imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));
 473
 474                        if (imxdma_chan_is_doing_cyclic(imxdmac))
 475                                /* Tasklet progression */
 476                                tasklet_schedule(&imxdmac->dma_tasklet);
 477
 478                        return;
 479                }
 480
 481                if (imxdma_hw_chain(imxdmac)) {
 482                        del_timer(&imxdmac->watchdog);
 483                        return;
 484                }
 485        }
 486
 487out:
 488        imx_dmav1_writel(imxdma, 0, DMA_CCR(chno));
 489        /* Tasklet irq */
 490        tasklet_schedule(&imxdmac->dma_tasklet);
 491}
 492
 493static irqreturn_t dma_irq_handler(int irq, void *dev_id)
 494{
 495        struct imxdma_engine *imxdma = dev_id;
 496        int i, disr;
 497
 498        if (!is_imx1_dma(imxdma))
 499                imxdma_err_handler(irq, dev_id);
 500
 501        disr = imx_dmav1_readl(imxdma, DMA_DISR);
 502
 503        dev_dbg(imxdma->dev, "%s called, disr=0x%08x\n", __func__, disr);
 504
 505        imx_dmav1_writel(imxdma, disr, DMA_DISR);
 506        for (i = 0; i < IMX_DMA_CHANNELS; i++) {
 507                if (disr & (1 << i))
 508                        dma_irq_handle_channel(&imxdma->channel[i]);
 509        }
 510
 511        return IRQ_HANDLED;
 512}
 513
 514static int imxdma_xfer_desc(struct imxdma_desc *d)
 515{
 516        struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);
 517        struct imxdma_engine *imxdma = imxdmac->imxdma;
 518        int slot = -1;
 519        int i;
 520
 521        /* Configure and enable */
 522        switch (d->type) {
 523        case IMXDMA_DESC_INTERLEAVED:
 524                /* Try to get a free 2D slot */
 525                for (i = 0; i < IMX_DMA_2D_SLOTS; i++) {
 526                        if ((imxdma->slots_2d[i].count > 0) &&
 527                        ((imxdma->slots_2d[i].xsr != d->x) ||
 528                        (imxdma->slots_2d[i].ysr != d->y) ||
 529                        (imxdma->slots_2d[i].wsr != d->w)))
 530                                continue;
 531                        slot = i;
 532                        break;
 533                }
 534                if (slot < 0)
 535                        return -EBUSY;
 536
 537                imxdma->slots_2d[slot].xsr = d->x;
 538                imxdma->slots_2d[slot].ysr = d->y;
 539                imxdma->slots_2d[slot].wsr = d->w;
 540                imxdma->slots_2d[slot].count++;
 541
 542                imxdmac->slot_2d = slot;
 543                imxdmac->enabled_2d = true;
 544
 545                if (slot == IMX_DMA_2D_SLOT_A) {
 546                        d->config_mem &= ~CCR_MSEL_B;
 547                        d->config_port &= ~CCR_MSEL_B;
 548                        imx_dmav1_writel(imxdma, d->x, DMA_XSRA);
 549                        imx_dmav1_writel(imxdma, d->y, DMA_YSRA);
 550                        imx_dmav1_writel(imxdma, d->w, DMA_WSRA);
 551                } else {
 552                        d->config_mem |= CCR_MSEL_B;
 553                        d->config_port |= CCR_MSEL_B;
 554                        imx_dmav1_writel(imxdma, d->x, DMA_XSRB);
 555                        imx_dmav1_writel(imxdma, d->y, DMA_YSRB);
 556                        imx_dmav1_writel(imxdma, d->w, DMA_WSRB);
 557                }
 558                /*
 559                 * We fall-through here intentionally, since a 2D transfer is
 560                 * similar to MEMCPY just adding the 2D slot configuration.
 561                 */
 562        case IMXDMA_DESC_MEMCPY:
 563                imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));
 564                imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));
 565                imx_dmav1_writel(imxdma, d->config_mem | (d->config_port << 2),
 566                         DMA_CCR(imxdmac->channel));
 567
 568                imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));
 569
 570                dev_dbg(imxdma->dev,
 571                        "%s channel: %d dest=0x%08llx src=0x%08llx dma_length=%zu\n",
 572                        __func__, imxdmac->channel,
 573                        (unsigned long long)d->dest,
 574                        (unsigned long long)d->src, d->len);
 575
 576                break;
 577        /* Cyclic transfer is the same as slave_sg with special sg configuration. */
 578        case IMXDMA_DESC_CYCLIC:
 579        case IMXDMA_DESC_SLAVE_SG:
 580                if (d->direction == DMA_DEV_TO_MEM) {
 581                        imx_dmav1_writel(imxdma, imxdmac->per_address,
 582                                         DMA_SAR(imxdmac->channel));
 583                        imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,
 584                                         DMA_CCR(imxdmac->channel));
 585
 586                        dev_dbg(imxdma->dev,
 587                                "%s channel: %d sg=%p sgcount=%d total length=%zu dev_addr=0x%08llx (dev2mem)\n",
 588                                __func__, imxdmac->channel,
 589                                d->sg, d->sgcount, d->len,
 590                                (unsigned long long)imxdmac->per_address);
 591                } else if (d->direction == DMA_MEM_TO_DEV) {
 592                        imx_dmav1_writel(imxdma, imxdmac->per_address,
 593                                         DMA_DAR(imxdmac->channel));
 594                        imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,
 595                                         DMA_CCR(imxdmac->channel));
 596
 597                        dev_dbg(imxdma->dev,
 598                                "%s channel: %d sg=%p sgcount=%d total length=%zu dev_addr=0x%08llx (mem2dev)\n",
 599                                __func__, imxdmac->channel,
 600                                d->sg, d->sgcount, d->len,
 601                                (unsigned long long)imxdmac->per_address);
 602                } else {
 603                        dev_err(imxdma->dev, "%s channel: %d bad dma mode\n",
 604                                __func__, imxdmac->channel);
 605                        return -EINVAL;
 606                }
 607
 608                imxdma_sg_next(d);
 609
 610                break;
 611        default:
 612                return -EINVAL;
 613        }
 614        imxdma_enable_hw(d);
 615        return 0;
 616}
 617
 618static void imxdma_tasklet(unsigned long data)
 619{
 620        struct imxdma_channel *imxdmac = (void *)data;
 621        struct imxdma_engine *imxdma = imxdmac->imxdma;
 622        struct imxdma_desc *desc;
 623        unsigned long flags;
 624
 625        spin_lock_irqsave(&imxdma->lock, flags);
 626
 627        if (list_empty(&imxdmac->ld_active)) {
 628                /* Someone might have called terminate all */
 629                spin_unlock_irqrestore(&imxdma->lock, flags);
 630                return;
 631        }
 632        desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);
 633
 634        /* If we are dealing with a cyclic descriptor, keep it on ld_active
 635         * and dont mark the descriptor as complete.
 636         * Only in non-cyclic cases it would be marked as complete
 637         */
 638        if (imxdma_chan_is_doing_cyclic(imxdmac))
 639                goto out;
 640        else
 641                dma_cookie_complete(&desc->desc);
 642
 643        /* Free 2D slot if it was an interleaved transfer */
 644        if (imxdmac->enabled_2d) {
 645                imxdma->slots_2d[imxdmac->slot_2d].count--;
 646                imxdmac->enabled_2d = false;
 647        }
 648
 649        list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);
 650
 651        if (!list_empty(&imxdmac->ld_queue)) {
 652                desc = list_first_entry(&imxdmac->ld_queue, struct imxdma_desc,
 653                                        node);
 654                list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);
 655                if (imxdma_xfer_desc(desc) < 0)
 656                        dev_warn(imxdma->dev, "%s: channel: %d couldn't xfer desc\n",
 657                                 __func__, imxdmac->channel);
 658        }
 659out:
 660        spin_unlock_irqrestore(&imxdma->lock, flags);
 661
 662        if (desc->desc.callback)
 663                desc->desc.callback(desc->desc.callback_param);
 664
 665}
 666
 667static int imxdma_terminate_all(struct dma_chan *chan)
 668{
 669        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 670        struct imxdma_engine *imxdma = imxdmac->imxdma;
 671        unsigned long flags;
 672
 673        imxdma_disable_hw(imxdmac);
 674
 675        spin_lock_irqsave(&imxdma->lock, flags);
 676        list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);
 677        list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);
 678        spin_unlock_irqrestore(&imxdma->lock, flags);
 679        return 0;
 680}
 681
 682static int imxdma_config(struct dma_chan *chan,
 683                         struct dma_slave_config *dmaengine_cfg)
 684{
 685        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 686        struct imxdma_engine *imxdma = imxdmac->imxdma;
 687        unsigned int mode = 0;
 688
 689        if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) {
 690                imxdmac->per_address = dmaengine_cfg->src_addr;
 691                imxdmac->watermark_level = dmaengine_cfg->src_maxburst;
 692                imxdmac->word_size = dmaengine_cfg->src_addr_width;
 693        } else {
 694                imxdmac->per_address = dmaengine_cfg->dst_addr;
 695                imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;
 696                imxdmac->word_size = dmaengine_cfg->dst_addr_width;
 697        }
 698
 699        switch (imxdmac->word_size) {
 700        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 701                mode = IMX_DMA_MEMSIZE_8;
 702                break;
 703        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 704                mode = IMX_DMA_MEMSIZE_16;
 705                break;
 706        default:
 707        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 708                mode = IMX_DMA_MEMSIZE_32;
 709                break;
 710        }
 711
 712        imxdmac->hw_chaining = 0;
 713
 714        imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |
 715                ((IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR) << 2) |
 716                CCR_REN;
 717        imxdmac->ccr_to_device =
 718                (IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR) |
 719                ((mode | IMX_DMA_TYPE_FIFO) << 2) | CCR_REN;
 720        imx_dmav1_writel(imxdma, imxdmac->dma_request,
 721                         DMA_RSSR(imxdmac->channel));
 722
 723        /* Set burst length */
 724        imx_dmav1_writel(imxdma, imxdmac->watermark_level *
 725                         imxdmac->word_size, DMA_BLR(imxdmac->channel));
 726
 727        return 0;
 728}
 729
 730static enum dma_status imxdma_tx_status(struct dma_chan *chan,
 731                                            dma_cookie_t cookie,
 732                                            struct dma_tx_state *txstate)
 733{
 734        return dma_cookie_status(chan, cookie, txstate);
 735}
 736
 737static dma_cookie_t imxdma_tx_submit(struct dma_async_tx_descriptor *tx)
 738{
 739        struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);
 740        struct imxdma_engine *imxdma = imxdmac->imxdma;
 741        dma_cookie_t cookie;
 742        unsigned long flags;
 743
 744        spin_lock_irqsave(&imxdma->lock, flags);
 745        list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);
 746        cookie = dma_cookie_assign(tx);
 747        spin_unlock_irqrestore(&imxdma->lock, flags);
 748
 749        return cookie;
 750}
 751
 752static int imxdma_alloc_chan_resources(struct dma_chan *chan)
 753{
 754        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 755        struct imx_dma_data *data = chan->private;
 756
 757        if (data != NULL)
 758                imxdmac->dma_request = data->dma_request;
 759
 760        while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {
 761                struct imxdma_desc *desc;
 762
 763                desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 764                if (!desc)
 765                        break;
 766                __memzero(&desc->desc, sizeof(struct dma_async_tx_descriptor));
 767                dma_async_tx_descriptor_init(&desc->desc, chan);
 768                desc->desc.tx_submit = imxdma_tx_submit;
 769                /* txd.flags will be overwritten in prep funcs */
 770                desc->desc.flags = DMA_CTRL_ACK;
 771                desc->status = DMA_COMPLETE;
 772
 773                list_add_tail(&desc->node, &imxdmac->ld_free);
 774                imxdmac->descs_allocated++;
 775        }
 776
 777        if (!imxdmac->descs_allocated)
 778                return -ENOMEM;
 779
 780        return imxdmac->descs_allocated;
 781}
 782
 783static void imxdma_free_chan_resources(struct dma_chan *chan)
 784{
 785        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 786        struct imxdma_engine *imxdma = imxdmac->imxdma;
 787        struct imxdma_desc *desc, *_desc;
 788        unsigned long flags;
 789
 790        spin_lock_irqsave(&imxdma->lock, flags);
 791
 792        imxdma_disable_hw(imxdmac);
 793        list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);
 794        list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);
 795
 796        spin_unlock_irqrestore(&imxdma->lock, flags);
 797
 798        list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {
 799                kfree(desc);
 800                imxdmac->descs_allocated--;
 801        }
 802        INIT_LIST_HEAD(&imxdmac->ld_free);
 803
 804        kfree(imxdmac->sg_list);
 805        imxdmac->sg_list = NULL;
 806}
 807
 808static struct dma_async_tx_descriptor *imxdma_prep_slave_sg(
 809                struct dma_chan *chan, struct scatterlist *sgl,
 810                unsigned int sg_len, enum dma_transfer_direction direction,
 811                unsigned long flags, void *context)
 812{
 813        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 814        struct scatterlist *sg;
 815        int i, dma_length = 0;
 816        struct imxdma_desc *desc;
 817
 818        if (list_empty(&imxdmac->ld_free) ||
 819            imxdma_chan_is_doing_cyclic(imxdmac))
 820                return NULL;
 821
 822        desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
 823
 824        for_each_sg(sgl, sg, sg_len, i) {
 825                dma_length += sg_dma_len(sg);
 826        }
 827
 828        switch (imxdmac->word_size) {
 829        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 830                if (sg_dma_len(sgl) & 3 || sgl->dma_address & 3)
 831                        return NULL;
 832                break;
 833        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 834                if (sg_dma_len(sgl) & 1 || sgl->dma_address & 1)
 835                        return NULL;
 836                break;
 837        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 838                break;
 839        default:
 840                return NULL;
 841        }
 842
 843        desc->type = IMXDMA_DESC_SLAVE_SG;
 844        desc->sg = sgl;
 845        desc->sgcount = sg_len;
 846        desc->len = dma_length;
 847        desc->direction = direction;
 848        if (direction == DMA_DEV_TO_MEM) {
 849                desc->src = imxdmac->per_address;
 850        } else {
 851                desc->dest = imxdmac->per_address;
 852        }
 853        desc->desc.callback = NULL;
 854        desc->desc.callback_param = NULL;
 855
 856        return &desc->desc;
 857}
 858
 859static struct dma_async_tx_descriptor *imxdma_prep_dma_cyclic(
 860                struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
 861                size_t period_len, enum dma_transfer_direction direction,
 862                unsigned long flags)
 863{
 864        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 865        struct imxdma_engine *imxdma = imxdmac->imxdma;
 866        struct imxdma_desc *desc;
 867        int i;
 868        unsigned int periods = buf_len / period_len;
 869
 870        dev_dbg(imxdma->dev, "%s channel: %d buf_len=%zu period_len=%zu\n",
 871                        __func__, imxdmac->channel, buf_len, period_len);
 872
 873        if (list_empty(&imxdmac->ld_free) ||
 874            imxdma_chan_is_doing_cyclic(imxdmac))
 875                return NULL;
 876
 877        desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
 878
 879        kfree(imxdmac->sg_list);
 880
 881        imxdmac->sg_list = kcalloc(periods + 1,
 882                        sizeof(struct scatterlist), GFP_ATOMIC);
 883        if (!imxdmac->sg_list)
 884                return NULL;
 885
 886        sg_init_table(imxdmac->sg_list, periods);
 887
 888        for (i = 0; i < periods; i++) {
 889                imxdmac->sg_list[i].page_link = 0;
 890                imxdmac->sg_list[i].offset = 0;
 891                imxdmac->sg_list[i].dma_address = dma_addr;
 892                sg_dma_len(&imxdmac->sg_list[i]) = period_len;
 893                dma_addr += period_len;
 894        }
 895
 896        /* close the loop */
 897        imxdmac->sg_list[periods].offset = 0;
 898        sg_dma_len(&imxdmac->sg_list[periods]) = 0;
 899        imxdmac->sg_list[periods].page_link =
 900                ((unsigned long)imxdmac->sg_list | 0x01) & ~0x02;
 901
 902        desc->type = IMXDMA_DESC_CYCLIC;
 903        desc->sg = imxdmac->sg_list;
 904        desc->sgcount = periods;
 905        desc->len = IMX_DMA_LENGTH_LOOP;
 906        desc->direction = direction;
 907        if (direction == DMA_DEV_TO_MEM) {
 908                desc->src = imxdmac->per_address;
 909        } else {
 910                desc->dest = imxdmac->per_address;
 911        }
 912        desc->desc.callback = NULL;
 913        desc->desc.callback_param = NULL;
 914
 915        return &desc->desc;
 916}
 917
 918static struct dma_async_tx_descriptor *imxdma_prep_dma_memcpy(
 919        struct dma_chan *chan, dma_addr_t dest,
 920        dma_addr_t src, size_t len, unsigned long flags)
 921{
 922        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 923        struct imxdma_engine *imxdma = imxdmac->imxdma;
 924        struct imxdma_desc *desc;
 925
 926        dev_dbg(imxdma->dev, "%s channel: %d src=0x%llx dst=0x%llx len=%zu\n",
 927                __func__, imxdmac->channel, (unsigned long long)src,
 928                (unsigned long long)dest, len);
 929
 930        if (list_empty(&imxdmac->ld_free) ||
 931            imxdma_chan_is_doing_cyclic(imxdmac))
 932                return NULL;
 933
 934        desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
 935
 936        desc->type = IMXDMA_DESC_MEMCPY;
 937        desc->src = src;
 938        desc->dest = dest;
 939        desc->len = len;
 940        desc->direction = DMA_MEM_TO_MEM;
 941        desc->config_port = IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR;
 942        desc->config_mem = IMX_DMA_MEMSIZE_32 | IMX_DMA_TYPE_LINEAR;
 943        desc->desc.callback = NULL;
 944        desc->desc.callback_param = NULL;
 945
 946        return &desc->desc;
 947}
 948
 949static struct dma_async_tx_descriptor *imxdma_prep_dma_interleaved(
 950        struct dma_chan *chan, struct dma_interleaved_template *xt,
 951        unsigned long flags)
 952{
 953        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 954        struct imxdma_engine *imxdma = imxdmac->imxdma;
 955        struct imxdma_desc *desc;
 956
 957        dev_dbg(imxdma->dev, "%s channel: %d src_start=0x%llx dst_start=0x%llx\n"
 958                "   src_sgl=%s dst_sgl=%s numf=%zu frame_size=%zu\n", __func__,
 959                imxdmac->channel, (unsigned long long)xt->src_start,
 960                (unsigned long long) xt->dst_start,
 961                xt->src_sgl ? "true" : "false", xt->dst_sgl ? "true" : "false",
 962                xt->numf, xt->frame_size);
 963
 964        if (list_empty(&imxdmac->ld_free) ||
 965            imxdma_chan_is_doing_cyclic(imxdmac))
 966                return NULL;
 967
 968        if (xt->frame_size != 1 || xt->numf <= 0 || xt->dir != DMA_MEM_TO_MEM)
 969                return NULL;
 970
 971        desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);
 972
 973        desc->type = IMXDMA_DESC_INTERLEAVED;
 974        desc->src = xt->src_start;
 975        desc->dest = xt->dst_start;
 976        desc->x = xt->sgl[0].size;
 977        desc->y = xt->numf;
 978        desc->w = xt->sgl[0].icg + desc->x;
 979        desc->len = desc->x * desc->y;
 980        desc->direction = DMA_MEM_TO_MEM;
 981        desc->config_port = IMX_DMA_MEMSIZE_32;
 982        desc->config_mem = IMX_DMA_MEMSIZE_32;
 983        if (xt->src_sgl)
 984                desc->config_mem |= IMX_DMA_TYPE_2D;
 985        if (xt->dst_sgl)
 986                desc->config_port |= IMX_DMA_TYPE_2D;
 987        desc->desc.callback = NULL;
 988        desc->desc.callback_param = NULL;
 989
 990        return &desc->desc;
 991}
 992
 993static void imxdma_issue_pending(struct dma_chan *chan)
 994{
 995        struct imxdma_channel *imxdmac = to_imxdma_chan(chan);
 996        struct imxdma_engine *imxdma = imxdmac->imxdma;
 997        struct imxdma_desc *desc;
 998        unsigned long flags;
 999
1000        spin_lock_irqsave(&imxdma->lock, flags);
1001        if (list_empty(&imxdmac->ld_active) &&
1002            !list_empty(&imxdmac->ld_queue)) {
1003                desc = list_first_entry(&imxdmac->ld_queue,
1004                                        struct imxdma_desc, node);
1005
1006                if (imxdma_xfer_desc(desc) < 0) {
1007                        dev_warn(imxdma->dev,
1008                                 "%s: channel: %d couldn't issue DMA xfer\n",
1009                                 __func__, imxdmac->channel);
1010                } else {
1011                        list_move_tail(imxdmac->ld_queue.next,
1012                                       &imxdmac->ld_active);
1013                }
1014        }
1015        spin_unlock_irqrestore(&imxdma->lock, flags);
1016}
1017
1018static bool imxdma_filter_fn(struct dma_chan *chan, void *param)
1019{
1020        struct imxdma_filter_data *fdata = param;
1021        struct imxdma_channel *imxdma_chan = to_imxdma_chan(chan);
1022
1023        if (chan->device->dev != fdata->imxdma->dev)
1024                return false;
1025
1026        imxdma_chan->dma_request = fdata->request;
1027        chan->private = NULL;
1028
1029        return true;
1030}
1031
1032static struct dma_chan *imxdma_xlate(struct of_phandle_args *dma_spec,
1033                                                struct of_dma *ofdma)
1034{
1035        int count = dma_spec->args_count;
1036        struct imxdma_engine *imxdma = ofdma->of_dma_data;
1037        struct imxdma_filter_data fdata = {
1038                .imxdma = imxdma,
1039        };
1040
1041        if (count != 1)
1042                return NULL;
1043
1044        fdata.request = dma_spec->args[0];
1045
1046        return dma_request_channel(imxdma->dma_device.cap_mask,
1047                                        imxdma_filter_fn, &fdata);
1048}
1049
1050static int __init imxdma_probe(struct platform_device *pdev)
1051        {
1052        struct imxdma_engine *imxdma;
1053        struct resource *res;
1054        const struct of_device_id *of_id;
1055        int ret, i;
1056        int irq, irq_err;
1057
1058        of_id = of_match_device(imx_dma_of_dev_id, &pdev->dev);
1059        if (of_id)
1060                pdev->id_entry = of_id->data;
1061
1062        imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL);
1063        if (!imxdma)
1064                return -ENOMEM;
1065
1066        imxdma->dev = &pdev->dev;
1067        imxdma->devtype = pdev->id_entry->driver_data;
1068
1069        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1070        imxdma->base = devm_ioremap_resource(&pdev->dev, res);
1071        if (IS_ERR(imxdma->base))
1072                return PTR_ERR(imxdma->base);
1073
1074        irq = platform_get_irq(pdev, 0);
1075        if (irq < 0)
1076                return irq;
1077
1078        imxdma->dma_ipg = devm_clk_get(&pdev->dev, "ipg");
1079        if (IS_ERR(imxdma->dma_ipg))
1080                return PTR_ERR(imxdma->dma_ipg);
1081
1082        imxdma->dma_ahb = devm_clk_get(&pdev->dev, "ahb");
1083        if (IS_ERR(imxdma->dma_ahb))
1084                return PTR_ERR(imxdma->dma_ahb);
1085
1086        clk_prepare_enable(imxdma->dma_ipg);
1087        clk_prepare_enable(imxdma->dma_ahb);
1088
1089        /* reset DMA module */
1090        imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR);
1091
1092        if (is_imx1_dma(imxdma)) {
1093                ret = devm_request_irq(&pdev->dev, irq,
1094                                       dma_irq_handler, 0, "DMA", imxdma);
1095                if (ret) {
1096                        dev_warn(imxdma->dev, "Can't register IRQ for DMA\n");
1097                        goto err;
1098                }
1099
1100                irq_err = platform_get_irq(pdev, 1);
1101                if (irq_err < 0) {
1102                        ret = irq_err;
1103                        goto err;
1104                }
1105
1106                ret = devm_request_irq(&pdev->dev, irq_err,
1107                                       imxdma_err_handler, 0, "DMA", imxdma);
1108                if (ret) {
1109                        dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n");
1110                        goto err;
1111                }
1112        }
1113
1114        /* enable DMA module */
1115        imx_dmav1_writel(imxdma, DCR_DEN, DMA_DCR);
1116
1117        /* clear all interrupts */
1118        imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DISR);
1119
1120        /* disable interrupts */
1121        imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DIMR);
1122
1123        INIT_LIST_HEAD(&imxdma->dma_device.channels);
1124
1125        dma_cap_set(DMA_SLAVE, imxdma->dma_device.cap_mask);
1126        dma_cap_set(DMA_CYCLIC, imxdma->dma_device.cap_mask);
1127        dma_cap_set(DMA_MEMCPY, imxdma->dma_device.cap_mask);
1128        dma_cap_set(DMA_INTERLEAVE, imxdma->dma_device.cap_mask);
1129
1130        /* Initialize 2D global parameters */
1131        for (i = 0; i < IMX_DMA_2D_SLOTS; i++)
1132                imxdma->slots_2d[i].count = 0;
1133
1134        spin_lock_init(&imxdma->lock);
1135
1136        /* Initialize channel parameters */
1137        for (i = 0; i < IMX_DMA_CHANNELS; i++) {
1138                struct imxdma_channel *imxdmac = &imxdma->channel[i];
1139
1140                if (!is_imx1_dma(imxdma)) {
1141                        ret = devm_request_irq(&pdev->dev, irq + i,
1142                                        dma_irq_handler, 0, "DMA", imxdma);
1143                        if (ret) {
1144                                dev_warn(imxdma->dev, "Can't register IRQ %d "
1145                                         "for DMA channel %d\n",
1146                                         irq + i, i);
1147                                goto err;
1148                        }
1149                        init_timer(&imxdmac->watchdog);
1150                        imxdmac->watchdog.function = &imxdma_watchdog;
1151                        imxdmac->watchdog.data = (unsigned long)imxdmac;
1152                }
1153
1154                imxdmac->imxdma = imxdma;
1155
1156                INIT_LIST_HEAD(&imxdmac->ld_queue);
1157                INIT_LIST_HEAD(&imxdmac->ld_free);
1158                INIT_LIST_HEAD(&imxdmac->ld_active);
1159
1160                tasklet_init(&imxdmac->dma_tasklet, imxdma_tasklet,
1161                             (unsigned long)imxdmac);
1162                imxdmac->chan.device = &imxdma->dma_device;
1163                dma_cookie_init(&imxdmac->chan);
1164                imxdmac->channel = i;
1165
1166                /* Add the channel to the DMAC list */
1167                list_add_tail(&imxdmac->chan.device_node,
1168                              &imxdma->dma_device.channels);
1169        }
1170
1171        imxdma->dma_device.dev = &pdev->dev;
1172
1173        imxdma->dma_device.device_alloc_chan_resources = imxdma_alloc_chan_resources;
1174        imxdma->dma_device.device_free_chan_resources = imxdma_free_chan_resources;
1175        imxdma->dma_device.device_tx_status = imxdma_tx_status;
1176        imxdma->dma_device.device_prep_slave_sg = imxdma_prep_slave_sg;
1177        imxdma->dma_device.device_prep_dma_cyclic = imxdma_prep_dma_cyclic;
1178        imxdma->dma_device.device_prep_dma_memcpy = imxdma_prep_dma_memcpy;
1179        imxdma->dma_device.device_prep_interleaved_dma = imxdma_prep_dma_interleaved;
1180        imxdma->dma_device.device_config = imxdma_config;
1181        imxdma->dma_device.device_terminate_all = imxdma_terminate_all;
1182        imxdma->dma_device.device_issue_pending = imxdma_issue_pending;
1183
1184        platform_set_drvdata(pdev, imxdma);
1185
1186        imxdma->dma_device.copy_align = 2; /* 2^2 = 4 bytes alignment */
1187        imxdma->dma_device.dev->dma_parms = &imxdma->dma_parms;
1188        dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff);
1189
1190        ret = dma_async_device_register(&imxdma->dma_device);
1191        if (ret) {
1192                dev_err(&pdev->dev, "unable to register\n");
1193                goto err;
1194        }
1195
1196        if (pdev->dev.of_node) {
1197                ret = of_dma_controller_register(pdev->dev.of_node,
1198                                imxdma_xlate, imxdma);
1199                if (ret) {
1200                        dev_err(&pdev->dev, "unable to register of_dma_controller\n");
1201                        goto err_of_dma_controller;
1202                }
1203        }
1204
1205        return 0;
1206
1207err_of_dma_controller:
1208        dma_async_device_unregister(&imxdma->dma_device);
1209err:
1210        clk_disable_unprepare(imxdma->dma_ipg);
1211        clk_disable_unprepare(imxdma->dma_ahb);
1212        return ret;
1213}
1214
1215static int imxdma_remove(struct platform_device *pdev)
1216{
1217        struct imxdma_engine *imxdma = platform_get_drvdata(pdev);
1218
1219        dma_async_device_unregister(&imxdma->dma_device);
1220
1221        if (pdev->dev.of_node)
1222                of_dma_controller_free(pdev->dev.of_node);
1223
1224        clk_disable_unprepare(imxdma->dma_ipg);
1225        clk_disable_unprepare(imxdma->dma_ahb);
1226
1227        return 0;
1228}
1229
1230static struct platform_driver imxdma_driver = {
1231        .driver         = {
1232                .name   = "imx-dma",
1233                .of_match_table = imx_dma_of_dev_id,
1234        },
1235        .id_table       = imx_dma_devtype,
1236        .remove         = imxdma_remove,
1237};
1238
1239static int __init imxdma_module_init(void)
1240{
1241        return platform_driver_probe(&imxdma_driver, imxdma_probe);
1242}
1243subsys_initcall(imxdma_module_init);
1244
1245MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
1246MODULE_DESCRIPTION("i.MX dma driver");
1247MODULE_LICENSE("GPL");
1248