linux/drivers/dma/ti/edma.c
<<
>>
Prefs
   1/*
   2 * TI EDMA DMA engine driver
   3 *
   4 * Copyright 2012 Texas Instruments
   5 *
   6 * This program is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU General Public License as
   8 * published by the Free Software Foundation version 2.
   9 *
  10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11 * kind, whether express or implied; without even the implied warranty
  12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 */
  15
  16#include <linux/dmaengine.h>
  17#include <linux/dma-mapping.h>
  18#include <linux/bitmap.h>
  19#include <linux/err.h>
  20#include <linux/init.h>
  21#include <linux/interrupt.h>
  22#include <linux/list.h>
  23#include <linux/module.h>
  24#include <linux/platform_device.h>
  25#include <linux/slab.h>
  26#include <linux/spinlock.h>
  27#include <linux/of.h>
  28#include <linux/of_dma.h>
  29#include <linux/of_irq.h>
  30#include <linux/of_address.h>
  31#include <linux/of_device.h>
  32#include <linux/pm_runtime.h>
  33
  34#include <linux/platform_data/edma.h>
  35
  36#include "../dmaengine.h"
  37#include "../virt-dma.h"
  38
  39/* Offsets matching "struct edmacc_param" */
  40#define PARM_OPT                0x00
  41#define PARM_SRC                0x04
  42#define PARM_A_B_CNT            0x08
  43#define PARM_DST                0x0c
  44#define PARM_SRC_DST_BIDX       0x10
  45#define PARM_LINK_BCNTRLD       0x14
  46#define PARM_SRC_DST_CIDX       0x18
  47#define PARM_CCNT               0x1c
  48
  49#define PARM_SIZE               0x20
  50
  51/* Offsets for EDMA CC global channel registers and their shadows */
  52#define SH_ER                   0x00    /* 64 bits */
  53#define SH_ECR                  0x08    /* 64 bits */
  54#define SH_ESR                  0x10    /* 64 bits */
  55#define SH_CER                  0x18    /* 64 bits */
  56#define SH_EER                  0x20    /* 64 bits */
  57#define SH_EECR                 0x28    /* 64 bits */
  58#define SH_EESR                 0x30    /* 64 bits */
  59#define SH_SER                  0x38    /* 64 bits */
  60#define SH_SECR                 0x40    /* 64 bits */
  61#define SH_IER                  0x50    /* 64 bits */
  62#define SH_IECR                 0x58    /* 64 bits */
  63#define SH_IESR                 0x60    /* 64 bits */
  64#define SH_IPR                  0x68    /* 64 bits */
  65#define SH_ICR                  0x70    /* 64 bits */
  66#define SH_IEVAL                0x78
  67#define SH_QER                  0x80
  68#define SH_QEER                 0x84
  69#define SH_QEECR                0x88
  70#define SH_QEESR                0x8c
  71#define SH_QSER                 0x90
  72#define SH_QSECR                0x94
  73#define SH_SIZE                 0x200
  74
  75/* Offsets for EDMA CC global registers */
  76#define EDMA_REV                0x0000
  77#define EDMA_CCCFG              0x0004
  78#define EDMA_QCHMAP             0x0200  /* 8 registers */
  79#define EDMA_DMAQNUM            0x0240  /* 8 registers (4 on OMAP-L1xx) */
  80#define EDMA_QDMAQNUM           0x0260
  81#define EDMA_QUETCMAP           0x0280
  82#define EDMA_QUEPRI             0x0284
  83#define EDMA_EMR                0x0300  /* 64 bits */
  84#define EDMA_EMCR               0x0308  /* 64 bits */
  85#define EDMA_QEMR               0x0310
  86#define EDMA_QEMCR              0x0314
  87#define EDMA_CCERR              0x0318
  88#define EDMA_CCERRCLR           0x031c
  89#define EDMA_EEVAL              0x0320
  90#define EDMA_DRAE               0x0340  /* 4 x 64 bits*/
  91#define EDMA_QRAE               0x0380  /* 4 registers */
  92#define EDMA_QUEEVTENTRY        0x0400  /* 2 x 16 registers */
  93#define EDMA_QSTAT              0x0600  /* 2 registers */
  94#define EDMA_QWMTHRA            0x0620
  95#define EDMA_QWMTHRB            0x0624
  96#define EDMA_CCSTAT             0x0640
  97
  98#define EDMA_M                  0x1000  /* global channel registers */
  99#define EDMA_ECR                0x1008
 100#define EDMA_ECRH               0x100C
 101#define EDMA_SHADOW0            0x2000  /* 4 shadow regions */
 102#define EDMA_PARM               0x4000  /* PaRAM entries */
 103
 104#define PARM_OFFSET(param_no)   (EDMA_PARM + ((param_no) << 5))
 105
 106#define EDMA_DCHMAP             0x0100  /* 64 registers */
 107
 108/* CCCFG register */
 109#define GET_NUM_DMACH(x)        (x & 0x7) /* bits 0-2 */
 110#define GET_NUM_QDMACH(x)       ((x & 0x70) >> 4) /* bits 4-6 */
 111#define GET_NUM_PAENTRY(x)      ((x & 0x7000) >> 12) /* bits 12-14 */
 112#define GET_NUM_EVQUE(x)        ((x & 0x70000) >> 16) /* bits 16-18 */
 113#define GET_NUM_REGN(x)         ((x & 0x300000) >> 20) /* bits 20-21 */
 114#define CHMAP_EXIST             BIT(24)
 115
 116/* CCSTAT register */
 117#define EDMA_CCSTAT_ACTV        BIT(4)
 118
 119/*
 120 * Max of 20 segments per channel to conserve PaRAM slots
 121 * Also note that MAX_NR_SG should be atleast the no.of periods
 122 * that are required for ASoC, otherwise DMA prep calls will
 123 * fail. Today davinci-pcm is the only user of this driver and
 124 * requires atleast 17 slots, so we setup the default to 20.
 125 */
 126#define MAX_NR_SG               20
 127#define EDMA_MAX_SLOTS          MAX_NR_SG
 128#define EDMA_DESCRIPTORS        16
 129
 130#define EDMA_CHANNEL_ANY                -1      /* for edma_alloc_channel() */
 131#define EDMA_SLOT_ANY                   -1      /* for edma_alloc_slot() */
 132#define EDMA_CONT_PARAMS_ANY             1001
 133#define EDMA_CONT_PARAMS_FIXED_EXACT     1002
 134#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
 135
 136/*
 137 * 64bit array registers are split into two 32bit registers:
 138 * reg0: channel/event 0-31
 139 * reg1: channel/event 32-63
 140 *
 141 * bit 5 in the channel number tells the array index (0/1)
 142 * bit 0-4 (0x1f) is the bit offset within the register
 143 */
 144#define EDMA_REG_ARRAY_INDEX(channel)   ((channel) >> 5)
 145#define EDMA_CHANNEL_BIT(channel)       (BIT((channel) & 0x1f))
 146
 147/* PaRAM slots are laid out like this */
 148struct edmacc_param {
 149        u32 opt;
 150        u32 src;
 151        u32 a_b_cnt;
 152        u32 dst;
 153        u32 src_dst_bidx;
 154        u32 link_bcntrld;
 155        u32 src_dst_cidx;
 156        u32 ccnt;
 157} __packed;
 158
 159/* fields in edmacc_param.opt */
 160#define SAM             BIT(0)
 161#define DAM             BIT(1)
 162#define SYNCDIM         BIT(2)
 163#define STATIC          BIT(3)
 164#define EDMA_FWID       (0x07 << 8)
 165#define TCCMODE         BIT(11)
 166#define EDMA_TCC(t)     ((t) << 12)
 167#define TCINTEN         BIT(20)
 168#define ITCINTEN        BIT(21)
 169#define TCCHEN          BIT(22)
 170#define ITCCHEN         BIT(23)
 171
 172struct edma_pset {
 173        u32                             len;
 174        dma_addr_t                      addr;
 175        struct edmacc_param             param;
 176};
 177
 178struct edma_desc {
 179        struct virt_dma_desc            vdesc;
 180        struct list_head                node;
 181        enum dma_transfer_direction     direction;
 182        int                             cyclic;
 183        bool                            polled;
 184        int                             absync;
 185        int                             pset_nr;
 186        struct edma_chan                *echan;
 187        int                             processed;
 188
 189        /*
 190         * The following 4 elements are used for residue accounting.
 191         *
 192         * - processed_stat: the number of SG elements we have traversed
 193         * so far to cover accounting. This is updated directly to processed
 194         * during edma_callback and is always <= processed, because processed
 195         * refers to the number of pending transfer (programmed to EDMA
 196         * controller), where as processed_stat tracks number of transfers
 197         * accounted for so far.
 198         *
 199         * - residue: The amount of bytes we have left to transfer for this desc
 200         *
 201         * - residue_stat: The residue in bytes of data we have covered
 202         * so far for accounting. This is updated directly to residue
 203         * during callbacks to keep it current.
 204         *
 205         * - sg_len: Tracks the length of the current intermediate transfer,
 206         * this is required to update the residue during intermediate transfer
 207         * completion callback.
 208         */
 209        int                             processed_stat;
 210        u32                             sg_len;
 211        u32                             residue;
 212        u32                             residue_stat;
 213
 214        struct edma_pset                pset[0];
 215};
 216
 217struct edma_cc;
 218
 219struct edma_tc {
 220        struct device_node              *node;
 221        u16                             id;
 222};
 223
 224struct edma_chan {
 225        struct virt_dma_chan            vchan;
 226        struct list_head                node;
 227        struct edma_desc                *edesc;
 228        struct edma_cc                  *ecc;
 229        struct edma_tc                  *tc;
 230        int                             ch_num;
 231        bool                            alloced;
 232        bool                            hw_triggered;
 233        int                             slot[EDMA_MAX_SLOTS];
 234        int                             missed;
 235        struct dma_slave_config         cfg;
 236};
 237
 238struct edma_cc {
 239        struct device                   *dev;
 240        struct edma_soc_info            *info;
 241        void __iomem                    *base;
 242        int                             id;
 243        bool                            legacy_mode;
 244
 245        /* eDMA3 resource information */
 246        unsigned                        num_channels;
 247        unsigned                        num_qchannels;
 248        unsigned                        num_region;
 249        unsigned                        num_slots;
 250        unsigned                        num_tc;
 251        bool                            chmap_exist;
 252        enum dma_event_q                default_queue;
 253
 254        unsigned int                    ccint;
 255        unsigned int                    ccerrint;
 256
 257        /*
 258         * The slot_inuse bit for each PaRAM slot is clear unless the slot is
 259         * in use by Linux or if it is allocated to be used by DSP.
 260         */
 261        unsigned long *slot_inuse;
 262
 263        struct dma_device               dma_slave;
 264        struct dma_device               *dma_memcpy;
 265        struct edma_chan                *slave_chans;
 266        struct edma_tc                  *tc_list;
 267        int                             dummy_slot;
 268};
 269
 270/* dummy param set used to (re)initialize parameter RAM slots */
 271static const struct edmacc_param dummy_paramset = {
 272        .link_bcntrld = 0xffff,
 273        .ccnt = 1,
 274};
 275
 276#define EDMA_BINDING_LEGACY     0
 277#define EDMA_BINDING_TPCC       1
 278static const u32 edma_binding_type[] = {
 279        [EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
 280        [EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
 281};
 282
 283static const struct of_device_id edma_of_ids[] = {
 284        {
 285                .compatible = "ti,edma3",
 286                .data = &edma_binding_type[EDMA_BINDING_LEGACY],
 287        },
 288        {
 289                .compatible = "ti,edma3-tpcc",
 290                .data = &edma_binding_type[EDMA_BINDING_TPCC],
 291        },
 292        {}
 293};
 294MODULE_DEVICE_TABLE(of, edma_of_ids);
 295
 296static const struct of_device_id edma_tptc_of_ids[] = {
 297        { .compatible = "ti,edma3-tptc", },
 298        {}
 299};
 300MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
 301
 302static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
 303{
 304        return (unsigned int)__raw_readl(ecc->base + offset);
 305}
 306
 307static inline void edma_write(struct edma_cc *ecc, int offset, int val)
 308{
 309        __raw_writel(val, ecc->base + offset);
 310}
 311
 312static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
 313                               unsigned or)
 314{
 315        unsigned val = edma_read(ecc, offset);
 316
 317        val &= and;
 318        val |= or;
 319        edma_write(ecc, offset, val);
 320}
 321
 322static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
 323{
 324        unsigned val = edma_read(ecc, offset);
 325
 326        val &= and;
 327        edma_write(ecc, offset, val);
 328}
 329
 330static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
 331{
 332        unsigned val = edma_read(ecc, offset);
 333
 334        val |= or;
 335        edma_write(ecc, offset, val);
 336}
 337
 338static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
 339                                           int i)
 340{
 341        return edma_read(ecc, offset + (i << 2));
 342}
 343
 344static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
 345                                    unsigned val)
 346{
 347        edma_write(ecc, offset + (i << 2), val);
 348}
 349
 350static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
 351                                     unsigned and, unsigned or)
 352{
 353        edma_modify(ecc, offset + (i << 2), and, or);
 354}
 355
 356static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
 357                                 unsigned or)
 358{
 359        edma_or(ecc, offset + (i << 2), or);
 360}
 361
 362static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
 363                                  unsigned or)
 364{
 365        edma_or(ecc, offset + ((i * 2 + j) << 2), or);
 366}
 367
 368static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
 369                                     int j, unsigned val)
 370{
 371        edma_write(ecc, offset + ((i * 2 + j) << 2), val);
 372}
 373
 374static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
 375{
 376        return edma_read(ecc, EDMA_SHADOW0 + offset);
 377}
 378
 379static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
 380                                                   int offset, int i)
 381{
 382        return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
 383}
 384
 385static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
 386                                      unsigned val)
 387{
 388        edma_write(ecc, EDMA_SHADOW0 + offset, val);
 389}
 390
 391static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
 392                                            int i, unsigned val)
 393{
 394        edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
 395}
 396
 397static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
 398                                           int param_no)
 399{
 400        return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
 401}
 402
 403static inline void edma_param_write(struct edma_cc *ecc, int offset,
 404                                    int param_no, unsigned val)
 405{
 406        edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
 407}
 408
 409static inline void edma_param_modify(struct edma_cc *ecc, int offset,
 410                                     int param_no, unsigned and, unsigned or)
 411{
 412        edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
 413}
 414
 415static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
 416                                  unsigned and)
 417{
 418        edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
 419}
 420
 421static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
 422                                 unsigned or)
 423{
 424        edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
 425}
 426
 427static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
 428                                          int priority)
 429{
 430        int bit = queue_no * 4;
 431
 432        edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
 433}
 434
 435static void edma_set_chmap(struct edma_chan *echan, int slot)
 436{
 437        struct edma_cc *ecc = echan->ecc;
 438        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 439
 440        if (ecc->chmap_exist) {
 441                slot = EDMA_CHAN_SLOT(slot);
 442                edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
 443        }
 444}
 445
 446static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
 447{
 448        struct edma_cc *ecc = echan->ecc;
 449        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 450        int idx = EDMA_REG_ARRAY_INDEX(channel);
 451        int ch_bit = EDMA_CHANNEL_BIT(channel);
 452
 453        if (enable) {
 454                edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 455                edma_shadow0_write_array(ecc, SH_IESR, idx, ch_bit);
 456        } else {
 457                edma_shadow0_write_array(ecc, SH_IECR, idx, ch_bit);
 458        }
 459}
 460
 461/*
 462 * paRAM slot management functions
 463 */
 464static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
 465                            const struct edmacc_param *param)
 466{
 467        slot = EDMA_CHAN_SLOT(slot);
 468        if (slot >= ecc->num_slots)
 469                return;
 470        memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
 471}
 472
 473static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
 474                           struct edmacc_param *param)
 475{
 476        slot = EDMA_CHAN_SLOT(slot);
 477        if (slot >= ecc->num_slots)
 478                return -EINVAL;
 479        memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
 480
 481        return 0;
 482}
 483
 484/**
 485 * edma_alloc_slot - allocate DMA parameter RAM
 486 * @ecc: pointer to edma_cc struct
 487 * @slot: specific slot to allocate; negative for "any unused slot"
 488 *
 489 * This allocates a parameter RAM slot, initializing it to hold a
 490 * dummy transfer.  Slots allocated using this routine have not been
 491 * mapped to a hardware DMA channel, and will normally be used by
 492 * linking to them from a slot associated with a DMA channel.
 493 *
 494 * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
 495 * slots may be allocated on behalf of DSP firmware.
 496 *
 497 * Returns the number of the slot, else negative errno.
 498 */
 499static int edma_alloc_slot(struct edma_cc *ecc, int slot)
 500{
 501        if (slot >= 0) {
 502                slot = EDMA_CHAN_SLOT(slot);
 503                /* Requesting entry paRAM slot for a HW triggered channel. */
 504                if (ecc->chmap_exist && slot < ecc->num_channels)
 505                        slot = EDMA_SLOT_ANY;
 506        }
 507
 508        if (slot < 0) {
 509                if (ecc->chmap_exist)
 510                        slot = 0;
 511                else
 512                        slot = ecc->num_channels;
 513                for (;;) {
 514                        slot = find_next_zero_bit(ecc->slot_inuse,
 515                                                  ecc->num_slots,
 516                                                  slot);
 517                        if (slot == ecc->num_slots)
 518                                return -ENOMEM;
 519                        if (!test_and_set_bit(slot, ecc->slot_inuse))
 520                                break;
 521                }
 522        } else if (slot >= ecc->num_slots) {
 523                return -EINVAL;
 524        } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
 525                return -EBUSY;
 526        }
 527
 528        edma_write_slot(ecc, slot, &dummy_paramset);
 529
 530        return EDMA_CTLR_CHAN(ecc->id, slot);
 531}
 532
 533static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
 534{
 535        slot = EDMA_CHAN_SLOT(slot);
 536        if (slot >= ecc->num_slots)
 537                return;
 538
 539        edma_write_slot(ecc, slot, &dummy_paramset);
 540        clear_bit(slot, ecc->slot_inuse);
 541}
 542
 543/**
 544 * edma_link - link one parameter RAM slot to another
 545 * @ecc: pointer to edma_cc struct
 546 * @from: parameter RAM slot originating the link
 547 * @to: parameter RAM slot which is the link target
 548 *
 549 * The originating slot should not be part of any active DMA transfer.
 550 */
 551static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
 552{
 553        if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
 554                dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
 555
 556        from = EDMA_CHAN_SLOT(from);
 557        to = EDMA_CHAN_SLOT(to);
 558        if (from >= ecc->num_slots || to >= ecc->num_slots)
 559                return;
 560
 561        edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
 562                          PARM_OFFSET(to));
 563}
 564
 565/**
 566 * edma_get_position - returns the current transfer point
 567 * @ecc: pointer to edma_cc struct
 568 * @slot: parameter RAM slot being examined
 569 * @dst:  true selects the dest position, false the source
 570 *
 571 * Returns the position of the current active slot
 572 */
 573static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
 574                                    bool dst)
 575{
 576        u32 offs;
 577
 578        slot = EDMA_CHAN_SLOT(slot);
 579        offs = PARM_OFFSET(slot);
 580        offs += dst ? PARM_DST : PARM_SRC;
 581
 582        return edma_read(ecc, offs);
 583}
 584
 585/*
 586 * Channels with event associations will be triggered by their hardware
 587 * events, and channels without such associations will be triggered by
 588 * software.  (At this writing there is no interface for using software
 589 * triggers except with channels that don't support hardware triggers.)
 590 */
 591static void edma_start(struct edma_chan *echan)
 592{
 593        struct edma_cc *ecc = echan->ecc;
 594        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 595        int idx = EDMA_REG_ARRAY_INDEX(channel);
 596        int ch_bit = EDMA_CHANNEL_BIT(channel);
 597
 598        if (!echan->hw_triggered) {
 599                /* EDMA channels without event association */
 600                dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 601                        edma_shadow0_read_array(ecc, SH_ESR, idx));
 602                edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 603        } else {
 604                /* EDMA channel with event association */
 605                dev_dbg(ecc->dev, "ER%d %08x\n", idx,
 606                        edma_shadow0_read_array(ecc, SH_ER, idx));
 607                /* Clear any pending event or error */
 608                edma_write_array(ecc, EDMA_ECR, idx, ch_bit);
 609                edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 610                /* Clear any SER */
 611                edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 612                edma_shadow0_write_array(ecc, SH_EESR, idx, ch_bit);
 613                dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 614                        edma_shadow0_read_array(ecc, SH_EER, idx));
 615        }
 616}
 617
 618static void edma_stop(struct edma_chan *echan)
 619{
 620        struct edma_cc *ecc = echan->ecc;
 621        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 622        int idx = EDMA_REG_ARRAY_INDEX(channel);
 623        int ch_bit = EDMA_CHANNEL_BIT(channel);
 624
 625        edma_shadow0_write_array(ecc, SH_EECR, idx, ch_bit);
 626        edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 627        edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 628        edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 629
 630        /* clear possibly pending completion interrupt */
 631        edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 632
 633        dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 634                edma_shadow0_read_array(ecc, SH_EER, idx));
 635
 636        /* REVISIT:  consider guarding against inappropriate event
 637         * chaining by overwriting with dummy_paramset.
 638         */
 639}
 640
 641/*
 642 * Temporarily disable EDMA hardware events on the specified channel,
 643 * preventing them from triggering new transfers
 644 */
 645static void edma_pause(struct edma_chan *echan)
 646{
 647        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 648
 649        edma_shadow0_write_array(echan->ecc, SH_EECR,
 650                                 EDMA_REG_ARRAY_INDEX(channel),
 651                                 EDMA_CHANNEL_BIT(channel));
 652}
 653
 654/* Re-enable EDMA hardware events on the specified channel.  */
 655static void edma_resume(struct edma_chan *echan)
 656{
 657        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 658
 659        edma_shadow0_write_array(echan->ecc, SH_EESR,
 660                                 EDMA_REG_ARRAY_INDEX(channel),
 661                                 EDMA_CHANNEL_BIT(channel));
 662}
 663
 664static void edma_trigger_channel(struct edma_chan *echan)
 665{
 666        struct edma_cc *ecc = echan->ecc;
 667        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 668        int idx = EDMA_REG_ARRAY_INDEX(channel);
 669        int ch_bit = EDMA_CHANNEL_BIT(channel);
 670
 671        edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 672
 673        dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 674                edma_shadow0_read_array(ecc, SH_ESR, idx));
 675}
 676
 677static void edma_clean_channel(struct edma_chan *echan)
 678{
 679        struct edma_cc *ecc = echan->ecc;
 680        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 681        int idx = EDMA_REG_ARRAY_INDEX(channel);
 682        int ch_bit = EDMA_CHANNEL_BIT(channel);
 683
 684        dev_dbg(ecc->dev, "EMR%d %08x\n", idx,
 685                edma_read_array(ecc, EDMA_EMR, idx));
 686        edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 687        /* Clear the corresponding EMR bits */
 688        edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 689        /* Clear any SER */
 690        edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 691        edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
 692}
 693
 694/* Move channel to a specific event queue */
 695static void edma_assign_channel_eventq(struct edma_chan *echan,
 696                                       enum dma_event_q eventq_no)
 697{
 698        struct edma_cc *ecc = echan->ecc;
 699        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 700        int bit = (channel & 0x7) * 4;
 701
 702        /* default to low priority queue */
 703        if (eventq_no == EVENTQ_DEFAULT)
 704                eventq_no = ecc->default_queue;
 705        if (eventq_no >= ecc->num_tc)
 706                return;
 707
 708        eventq_no &= 7;
 709        edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
 710                          eventq_no << bit);
 711}
 712
 713static int edma_alloc_channel(struct edma_chan *echan,
 714                              enum dma_event_q eventq_no)
 715{
 716        struct edma_cc *ecc = echan->ecc;
 717        int channel = EDMA_CHAN_SLOT(echan->ch_num);
 718
 719        /* ensure access through shadow region 0 */
 720        edma_or_array2(ecc, EDMA_DRAE, 0, EDMA_REG_ARRAY_INDEX(channel),
 721                       EDMA_CHANNEL_BIT(channel));
 722
 723        /* ensure no events are pending */
 724        edma_stop(echan);
 725
 726        edma_setup_interrupt(echan, true);
 727
 728        edma_assign_channel_eventq(echan, eventq_no);
 729
 730        return 0;
 731}
 732
 733static void edma_free_channel(struct edma_chan *echan)
 734{
 735        /* ensure no events are pending */
 736        edma_stop(echan);
 737        /* REVISIT should probably take out of shadow region 0 */
 738        edma_setup_interrupt(echan, false);
 739}
 740
 741static inline struct edma_cc *to_edma_cc(struct dma_device *d)
 742{
 743        return container_of(d, struct edma_cc, dma_slave);
 744}
 745
 746static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 747{
 748        return container_of(c, struct edma_chan, vchan.chan);
 749}
 750
 751static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
 752{
 753        return container_of(tx, struct edma_desc, vdesc.tx);
 754}
 755
 756static void edma_desc_free(struct virt_dma_desc *vdesc)
 757{
 758        kfree(container_of(vdesc, struct edma_desc, vdesc));
 759}
 760
 761/* Dispatch a queued descriptor to the controller (caller holds lock) */
 762static void edma_execute(struct edma_chan *echan)
 763{
 764        struct edma_cc *ecc = echan->ecc;
 765        struct virt_dma_desc *vdesc;
 766        struct edma_desc *edesc;
 767        struct device *dev = echan->vchan.chan.device->dev;
 768        int i, j, left, nslots;
 769
 770        if (!echan->edesc) {
 771                /* Setup is needed for the first transfer */
 772                vdesc = vchan_next_desc(&echan->vchan);
 773                if (!vdesc)
 774                        return;
 775                list_del(&vdesc->node);
 776                echan->edesc = to_edma_desc(&vdesc->tx);
 777        }
 778
 779        edesc = echan->edesc;
 780
 781        /* Find out how many left */
 782        left = edesc->pset_nr - edesc->processed;
 783        nslots = min(MAX_NR_SG, left);
 784        edesc->sg_len = 0;
 785
 786        /* Write descriptor PaRAM set(s) */
 787        for (i = 0; i < nslots; i++) {
 788                j = i + edesc->processed;
 789                edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
 790                edesc->sg_len += edesc->pset[j].len;
 791                dev_vdbg(dev,
 792                         "\n pset[%d]:\n"
 793                         "  chnum\t%d\n"
 794                         "  slot\t%d\n"
 795                         "  opt\t%08x\n"
 796                         "  src\t%08x\n"
 797                         "  dst\t%08x\n"
 798                         "  abcnt\t%08x\n"
 799                         "  ccnt\t%08x\n"
 800                         "  bidx\t%08x\n"
 801                         "  cidx\t%08x\n"
 802                         "  lkrld\t%08x\n",
 803                         j, echan->ch_num, echan->slot[i],
 804                         edesc->pset[j].param.opt,
 805                         edesc->pset[j].param.src,
 806                         edesc->pset[j].param.dst,
 807                         edesc->pset[j].param.a_b_cnt,
 808                         edesc->pset[j].param.ccnt,
 809                         edesc->pset[j].param.src_dst_bidx,
 810                         edesc->pset[j].param.src_dst_cidx,
 811                         edesc->pset[j].param.link_bcntrld);
 812                /* Link to the previous slot if not the last set */
 813                if (i != (nslots - 1))
 814                        edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
 815        }
 816
 817        edesc->processed += nslots;
 818
 819        /*
 820         * If this is either the last set in a set of SG-list transactions
 821         * then setup a link to the dummy slot, this results in all future
 822         * events being absorbed and that's OK because we're done
 823         */
 824        if (edesc->processed == edesc->pset_nr) {
 825                if (edesc->cyclic)
 826                        edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
 827                else
 828                        edma_link(ecc, echan->slot[nslots - 1],
 829                                  echan->ecc->dummy_slot);
 830        }
 831
 832        if (echan->missed) {
 833                /*
 834                 * This happens due to setup times between intermediate
 835                 * transfers in long SG lists which have to be broken up into
 836                 * transfers of MAX_NR_SG
 837                 */
 838                dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 839                edma_clean_channel(echan);
 840                edma_stop(echan);
 841                edma_start(echan);
 842                edma_trigger_channel(echan);
 843                echan->missed = 0;
 844        } else if (edesc->processed <= MAX_NR_SG) {
 845                dev_dbg(dev, "first transfer starting on channel %d\n",
 846                        echan->ch_num);
 847                edma_start(echan);
 848        } else {
 849                dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 850                        echan->ch_num, edesc->processed);
 851                edma_resume(echan);
 852        }
 853}
 854
 855static int edma_terminate_all(struct dma_chan *chan)
 856{
 857        struct edma_chan *echan = to_edma_chan(chan);
 858        unsigned long flags;
 859        LIST_HEAD(head);
 860
 861        spin_lock_irqsave(&echan->vchan.lock, flags);
 862
 863        /*
 864         * Stop DMA activity: we assume the callback will not be called
 865         * after edma_dma() returns (even if it does, it will see
 866         * echan->edesc is NULL and exit.)
 867         */
 868        if (echan->edesc) {
 869                edma_stop(echan);
 870                /* Move the cyclic channel back to default queue */
 871                if (!echan->tc && echan->edesc->cyclic)
 872                        edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
 873
 874                vchan_terminate_vdesc(&echan->edesc->vdesc);
 875                echan->edesc = NULL;
 876        }
 877
 878        vchan_get_all_descriptors(&echan->vchan, &head);
 879        spin_unlock_irqrestore(&echan->vchan.lock, flags);
 880        vchan_dma_desc_free_list(&echan->vchan, &head);
 881
 882        return 0;
 883}
 884
 885static void edma_synchronize(struct dma_chan *chan)
 886{
 887        struct edma_chan *echan = to_edma_chan(chan);
 888
 889        vchan_synchronize(&echan->vchan);
 890}
 891
 892static int edma_slave_config(struct dma_chan *chan,
 893        struct dma_slave_config *cfg)
 894{
 895        struct edma_chan *echan = to_edma_chan(chan);
 896
 897        if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 898            cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 899                return -EINVAL;
 900
 901        if (cfg->src_maxburst > chan->device->max_burst ||
 902            cfg->dst_maxburst > chan->device->max_burst)
 903                return -EINVAL;
 904
 905        memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 906
 907        return 0;
 908}
 909
 910static int edma_dma_pause(struct dma_chan *chan)
 911{
 912        struct edma_chan *echan = to_edma_chan(chan);
 913
 914        if (!echan->edesc)
 915                return -EINVAL;
 916
 917        edma_pause(echan);
 918        return 0;
 919}
 920
 921static int edma_dma_resume(struct dma_chan *chan)
 922{
 923        struct edma_chan *echan = to_edma_chan(chan);
 924
 925        edma_resume(echan);
 926        return 0;
 927}
 928
 929/*
 930 * A PaRAM set configuration abstraction used by other modes
 931 * @chan: Channel who's PaRAM set we're configuring
 932 * @pset: PaRAM set to initialize and setup.
 933 * @src_addr: Source address of the DMA
 934 * @dst_addr: Destination address of the DMA
 935 * @burst: In units of dev_width, how much to send
 936 * @dev_width: How much is the dev_width
 937 * @dma_length: Total length of the DMA transfer
 938 * @direction: Direction of the transfer
 939 */
 940static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 941                            dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 942                            unsigned int acnt, unsigned int dma_length,
 943                            enum dma_transfer_direction direction)
 944{
 945        struct edma_chan *echan = to_edma_chan(chan);
 946        struct device *dev = chan->device->dev;
 947        struct edmacc_param *param = &epset->param;
 948        int bcnt, ccnt, cidx;
 949        int src_bidx, dst_bidx, src_cidx, dst_cidx;
 950        int absync;
 951
 952        /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 953        if (!burst)
 954                burst = 1;
 955        /*
 956         * If the maxburst is equal to the fifo width, use
 957         * A-synced transfers. This allows for large contiguous
 958         * buffer transfers using only one PaRAM set.
 959         */
 960        if (burst == 1) {
 961                /*
 962                 * For the A-sync case, bcnt and ccnt are the remainder
 963                 * and quotient respectively of the division of:
 964                 * (dma_length / acnt) by (SZ_64K -1). This is so
 965                 * that in case bcnt over flows, we have ccnt to use.
 966                 * Note: In A-sync tranfer only, bcntrld is used, but it
 967                 * only applies for sg_dma_len(sg) >= SZ_64K.
 968                 * In this case, the best way adopted is- bccnt for the
 969                 * first frame will be the remainder below. Then for
 970                 * every successive frame, bcnt will be SZ_64K-1. This
 971                 * is assured as bcntrld = 0xffff in end of function.
 972                 */
 973                absync = false;
 974                ccnt = dma_length / acnt / (SZ_64K - 1);
 975                bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 976                /*
 977                 * If bcnt is non-zero, we have a remainder and hence an
 978                 * extra frame to transfer, so increment ccnt.
 979                 */
 980                if (bcnt)
 981                        ccnt++;
 982                else
 983                        bcnt = SZ_64K - 1;
 984                cidx = acnt;
 985        } else {
 986                /*
 987                 * If maxburst is greater than the fifo address_width,
 988                 * use AB-synced transfers where A count is the fifo
 989                 * address_width and B count is the maxburst. In this
 990                 * case, we are limited to transfers of C count frames
 991                 * of (address_width * maxburst) where C count is limited
 992                 * to SZ_64K-1. This places an upper bound on the length
 993                 * of an SG segment that can be handled.
 994                 */
 995                absync = true;
 996                bcnt = burst;
 997                ccnt = dma_length / (acnt * bcnt);
 998                if (ccnt > (SZ_64K - 1)) {
 999                        dev_err(dev, "Exceeded max SG segment size\n");
1000                        return -EINVAL;
1001                }
1002                cidx = acnt * bcnt;
1003        }
1004
1005        epset->len = dma_length;
1006
1007        if (direction == DMA_MEM_TO_DEV) {
1008                src_bidx = acnt;
1009                src_cidx = cidx;
1010                dst_bidx = 0;
1011                dst_cidx = 0;
1012                epset->addr = src_addr;
1013        } else if (direction == DMA_DEV_TO_MEM)  {
1014                src_bidx = 0;
1015                src_cidx = 0;
1016                dst_bidx = acnt;
1017                dst_cidx = cidx;
1018                epset->addr = dst_addr;
1019        } else if (direction == DMA_MEM_TO_MEM)  {
1020                src_bidx = acnt;
1021                src_cidx = cidx;
1022                dst_bidx = acnt;
1023                dst_cidx = cidx;
1024                epset->addr = src_addr;
1025        } else {
1026                dev_err(dev, "%s: direction not implemented yet\n", __func__);
1027                return -EINVAL;
1028        }
1029
1030        param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
1031        /* Configure A or AB synchronized transfers */
1032        if (absync)
1033                param->opt |= SYNCDIM;
1034
1035        param->src = src_addr;
1036        param->dst = dst_addr;
1037
1038        param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1039        param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
1040
1041        param->a_b_cnt = bcnt << 16 | acnt;
1042        param->ccnt = ccnt;
1043        /*
1044         * Only time when (bcntrld) auto reload is required is for
1045         * A-sync case, and in this case, a requirement of reload value
1046         * of SZ_64K-1 only is assured. 'link' is initially set to NULL
1047         * and then later will be populated by edma_execute.
1048         */
1049        param->link_bcntrld = 0xffffffff;
1050        return absync;
1051}
1052
1053static struct dma_async_tx_descriptor *edma_prep_slave_sg(
1054        struct dma_chan *chan, struct scatterlist *sgl,
1055        unsigned int sg_len, enum dma_transfer_direction direction,
1056        unsigned long tx_flags, void *context)
1057{
1058        struct edma_chan *echan = to_edma_chan(chan);
1059        struct device *dev = chan->device->dev;
1060        struct edma_desc *edesc;
1061        dma_addr_t src_addr = 0, dst_addr = 0;
1062        enum dma_slave_buswidth dev_width;
1063        u32 burst;
1064        struct scatterlist *sg;
1065        int i, nslots, ret;
1066
1067        if (unlikely(!echan || !sgl || !sg_len))
1068                return NULL;
1069
1070        if (direction == DMA_DEV_TO_MEM) {
1071                src_addr = echan->cfg.src_addr;
1072                dev_width = echan->cfg.src_addr_width;
1073                burst = echan->cfg.src_maxburst;
1074        } else if (direction == DMA_MEM_TO_DEV) {
1075                dst_addr = echan->cfg.dst_addr;
1076                dev_width = echan->cfg.dst_addr_width;
1077                burst = echan->cfg.dst_maxburst;
1078        } else {
1079                dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1080                return NULL;
1081        }
1082
1083        if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1084                dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1085                return NULL;
1086        }
1087
1088        edesc = kzalloc(struct_size(edesc, pset, sg_len), GFP_ATOMIC);
1089        if (!edesc)
1090                return NULL;
1091
1092        edesc->pset_nr = sg_len;
1093        edesc->residue = 0;
1094        edesc->direction = direction;
1095        edesc->echan = echan;
1096
1097        /* Allocate a PaRAM slot, if needed */
1098        nslots = min_t(unsigned, MAX_NR_SG, sg_len);
1099
1100        for (i = 0; i < nslots; i++) {
1101                if (echan->slot[i] < 0) {
1102                        echan->slot[i] =
1103                                edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1104                        if (echan->slot[i] < 0) {
1105                                kfree(edesc);
1106                                dev_err(dev, "%s: Failed to allocate slot\n",
1107                                        __func__);
1108                                return NULL;
1109                        }
1110                }
1111        }
1112
1113        /* Configure PaRAM sets for each SG */
1114        for_each_sg(sgl, sg, sg_len, i) {
1115                /* Get address for each SG */
1116                if (direction == DMA_DEV_TO_MEM)
1117                        dst_addr = sg_dma_address(sg);
1118                else
1119                        src_addr = sg_dma_address(sg);
1120
1121                ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1122                                       dst_addr, burst, dev_width,
1123                                       sg_dma_len(sg), direction);
1124                if (ret < 0) {
1125                        kfree(edesc);
1126                        return NULL;
1127                }
1128
1129                edesc->absync = ret;
1130                edesc->residue += sg_dma_len(sg);
1131
1132                if (i == sg_len - 1)
1133                        /* Enable completion interrupt */
1134                        edesc->pset[i].param.opt |= TCINTEN;
1135                else if (!((i+1) % MAX_NR_SG))
1136                        /*
1137                         * Enable early completion interrupt for the
1138                         * intermediateset. In this case the driver will be
1139                         * notified when the paRAM set is submitted to TC. This
1140                         * will allow more time to set up the next set of slots.
1141                         */
1142                        edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);
1143        }
1144        edesc->residue_stat = edesc->residue;
1145
1146        return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1147}
1148
1149static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
1150        struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1151        size_t len, unsigned long tx_flags)
1152{
1153        int ret, nslots;
1154        struct edma_desc *edesc;
1155        struct device *dev = chan->device->dev;
1156        struct edma_chan *echan = to_edma_chan(chan);
1157        unsigned int width, pset_len, array_size;
1158
1159        if (unlikely(!echan || !len))
1160                return NULL;
1161
1162        /* Align the array size (acnt block) with the transfer properties */
1163        switch (__ffs((src | dest | len))) {
1164        case 0:
1165                array_size = SZ_32K - 1;
1166                break;
1167        case 1:
1168                array_size = SZ_32K - 2;
1169                break;
1170        default:
1171                array_size = SZ_32K - 4;
1172                break;
1173        }
1174
1175        if (len < SZ_64K) {
1176                /*
1177                 * Transfer size less than 64K can be handled with one paRAM
1178                 * slot and with one burst.
1179                 * ACNT = length
1180                 */
1181                width = len;
1182                pset_len = len;
1183                nslots = 1;
1184        } else {
1185                /*
1186                 * Transfer size bigger than 64K will be handled with maximum of
1187                 * two paRAM slots.
1188                 * slot1: (full_length / 32767) times 32767 bytes bursts.
1189                 *        ACNT = 32767, length1: (full_length / 32767) * 32767
1190                 * slot2: the remaining amount of data after slot1.
1191                 *        ACNT = full_length - length1, length2 = ACNT
1192                 *
1193                 * When the full_length is multibple of 32767 one slot can be
1194                 * used to complete the transfer.
1195                 */
1196                width = array_size;
1197                pset_len = rounddown(len, width);
1198                /* One slot is enough for lengths multiple of (SZ_32K -1) */
1199                if (unlikely(pset_len == len))
1200                        nslots = 1;
1201                else
1202                        nslots = 2;
1203        }
1204
1205        edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1206        if (!edesc)
1207                return NULL;
1208
1209        edesc->pset_nr = nslots;
1210        edesc->residue = edesc->residue_stat = len;
1211        edesc->direction = DMA_MEM_TO_MEM;
1212        edesc->echan = echan;
1213
1214        ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
1215                               width, pset_len, DMA_MEM_TO_MEM);
1216        if (ret < 0) {
1217                kfree(edesc);
1218                return NULL;
1219        }
1220
1221        edesc->absync = ret;
1222
1223        edesc->pset[0].param.opt |= ITCCHEN;
1224        if (nslots == 1) {
1225                /* Enable transfer complete interrupt if requested */
1226                if (tx_flags & DMA_PREP_INTERRUPT)
1227                        edesc->pset[0].param.opt |= TCINTEN;
1228        } else {
1229                /* Enable transfer complete chaining for the first slot */
1230                edesc->pset[0].param.opt |= TCCHEN;
1231
1232                if (echan->slot[1] < 0) {
1233                        echan->slot[1] = edma_alloc_slot(echan->ecc,
1234                                                         EDMA_SLOT_ANY);
1235                        if (echan->slot[1] < 0) {
1236                                kfree(edesc);
1237                                dev_err(dev, "%s: Failed to allocate slot\n",
1238                                        __func__);
1239                                return NULL;
1240                        }
1241                }
1242                dest += pset_len;
1243                src += pset_len;
1244                pset_len = width = len % array_size;
1245
1246                ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,
1247                                       width, pset_len, DMA_MEM_TO_MEM);
1248                if (ret < 0) {
1249                        kfree(edesc);
1250                        return NULL;
1251                }
1252
1253                edesc->pset[1].param.opt |= ITCCHEN;
1254                /* Enable transfer complete interrupt if requested */
1255                if (tx_flags & DMA_PREP_INTERRUPT)
1256                        edesc->pset[1].param.opt |= TCINTEN;
1257        }
1258
1259        if (!(tx_flags & DMA_PREP_INTERRUPT))
1260                edesc->polled = true;
1261
1262        return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1263}
1264
1265static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
1266        struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1267        size_t period_len, enum dma_transfer_direction direction,
1268        unsigned long tx_flags)
1269{
1270        struct edma_chan *echan = to_edma_chan(chan);
1271        struct device *dev = chan->device->dev;
1272        struct edma_desc *edesc;
1273        dma_addr_t src_addr, dst_addr;
1274        enum dma_slave_buswidth dev_width;
1275        bool use_intermediate = false;
1276        u32 burst;
1277        int i, ret, nslots;
1278
1279        if (unlikely(!echan || !buf_len || !period_len))
1280                return NULL;
1281
1282        if (direction == DMA_DEV_TO_MEM) {
1283                src_addr = echan->cfg.src_addr;
1284                dst_addr = buf_addr;
1285                dev_width = echan->cfg.src_addr_width;
1286                burst = echan->cfg.src_maxburst;
1287        } else if (direction == DMA_MEM_TO_DEV) {
1288                src_addr = buf_addr;
1289                dst_addr = echan->cfg.dst_addr;
1290                dev_width = echan->cfg.dst_addr_width;
1291                burst = echan->cfg.dst_maxburst;
1292        } else {
1293                dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1294                return NULL;
1295        }
1296
1297        if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1298                dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1299                return NULL;
1300        }
1301
1302        if (unlikely(buf_len % period_len)) {
1303                dev_err(dev, "Period should be multiple of Buffer length\n");
1304                return NULL;
1305        }
1306
1307        nslots = (buf_len / period_len) + 1;
1308
1309        /*
1310         * Cyclic DMA users such as audio cannot tolerate delays introduced
1311         * by cases where the number of periods is more than the maximum
1312         * number of SGs the EDMA driver can handle at a time. For DMA types
1313         * such as Slave SGs, such delays are tolerable and synchronized,
1314         * but the synchronization is difficult to achieve with Cyclic and
1315         * cannot be guaranteed, so we error out early.
1316         */
1317        if (nslots > MAX_NR_SG) {
1318                /*
1319                 * If the burst and period sizes are the same, we can put
1320                 * the full buffer into a single period and activate
1321                 * intermediate interrupts. This will produce interrupts
1322                 * after each burst, which is also after each desired period.
1323                 */
1324                if (burst == period_len) {
1325                        period_len = buf_len;
1326                        nslots = 2;
1327                        use_intermediate = true;
1328                } else {
1329                        return NULL;
1330                }
1331        }
1332
1333        edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1334        if (!edesc)
1335                return NULL;
1336
1337        edesc->cyclic = 1;
1338        edesc->pset_nr = nslots;
1339        edesc->residue = edesc->residue_stat = buf_len;
1340        edesc->direction = direction;
1341        edesc->echan = echan;
1342
1343        dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
1344                __func__, echan->ch_num, nslots, period_len, buf_len);
1345
1346        for (i = 0; i < nslots; i++) {
1347                /* Allocate a PaRAM slot, if needed */
1348                if (echan->slot[i] < 0) {
1349                        echan->slot[i] =
1350                                edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1351                        if (echan->slot[i] < 0) {
1352                                kfree(edesc);
1353                                dev_err(dev, "%s: Failed to allocate slot\n",
1354                                        __func__);
1355                                return NULL;
1356                        }
1357                }
1358
1359                if (i == nslots - 1) {
1360                        memcpy(&edesc->pset[i], &edesc->pset[0],
1361                               sizeof(edesc->pset[0]));
1362                        break;
1363                }
1364
1365                ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1366                                       dst_addr, burst, dev_width, period_len,
1367                                       direction);
1368                if (ret < 0) {
1369                        kfree(edesc);
1370                        return NULL;
1371                }
1372
1373                if (direction == DMA_DEV_TO_MEM)
1374                        dst_addr += period_len;
1375                else
1376                        src_addr += period_len;
1377
1378                dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
1379                dev_vdbg(dev,
1380                        "\n pset[%d]:\n"
1381                        "  chnum\t%d\n"
1382                        "  slot\t%d\n"
1383                        "  opt\t%08x\n"
1384                        "  src\t%08x\n"
1385                        "  dst\t%08x\n"
1386                        "  abcnt\t%08x\n"
1387                        "  ccnt\t%08x\n"
1388                        "  bidx\t%08x\n"
1389                        "  cidx\t%08x\n"
1390                        "  lkrld\t%08x\n",
1391                        i, echan->ch_num, echan->slot[i],
1392                        edesc->pset[i].param.opt,
1393                        edesc->pset[i].param.src,
1394                        edesc->pset[i].param.dst,
1395                        edesc->pset[i].param.a_b_cnt,
1396                        edesc->pset[i].param.ccnt,
1397                        edesc->pset[i].param.src_dst_bidx,
1398                        edesc->pset[i].param.src_dst_cidx,
1399                        edesc->pset[i].param.link_bcntrld);
1400
1401                edesc->absync = ret;
1402
1403                /*
1404                 * Enable period interrupt only if it is requested
1405                 */
1406                if (tx_flags & DMA_PREP_INTERRUPT) {
1407                        edesc->pset[i].param.opt |= TCINTEN;
1408
1409                        /* Also enable intermediate interrupts if necessary */
1410                        if (use_intermediate)
1411                                edesc->pset[i].param.opt |= ITCINTEN;
1412                }
1413        }
1414
1415        /* Place the cyclic channel to highest priority queue */
1416        if (!echan->tc)
1417                edma_assign_channel_eventq(echan, EVENTQ_0);
1418
1419        return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1420}
1421
1422static void edma_completion_handler(struct edma_chan *echan)
1423{
1424        struct device *dev = echan->vchan.chan.device->dev;
1425        struct edma_desc *edesc;
1426
1427        spin_lock(&echan->vchan.lock);
1428        edesc = echan->edesc;
1429        if (edesc) {
1430                if (edesc->cyclic) {
1431                        vchan_cyclic_callback(&edesc->vdesc);
1432                        spin_unlock(&echan->vchan.lock);
1433                        return;
1434                } else if (edesc->processed == edesc->pset_nr) {
1435                        edesc->residue = 0;
1436                        edma_stop(echan);
1437                        vchan_cookie_complete(&edesc->vdesc);
1438                        echan->edesc = NULL;
1439
1440                        dev_dbg(dev, "Transfer completed on channel %d\n",
1441                                echan->ch_num);
1442                } else {
1443                        dev_dbg(dev, "Sub transfer completed on channel %d\n",
1444                                echan->ch_num);
1445
1446                        edma_pause(echan);
1447
1448                        /* Update statistics for tx_status */
1449                        edesc->residue -= edesc->sg_len;
1450                        edesc->residue_stat = edesc->residue;
1451                        edesc->processed_stat = edesc->processed;
1452                }
1453                edma_execute(echan);
1454        }
1455
1456        spin_unlock(&echan->vchan.lock);
1457}
1458
1459/* eDMA interrupt handler */
1460static irqreturn_t dma_irq_handler(int irq, void *data)
1461{
1462        struct edma_cc *ecc = data;
1463        int ctlr;
1464        u32 sh_ier;
1465        u32 sh_ipr;
1466        u32 bank;
1467
1468        ctlr = ecc->id;
1469        if (ctlr < 0)
1470                return IRQ_NONE;
1471
1472        dev_vdbg(ecc->dev, "dma_irq_handler\n");
1473
1474        sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1475        if (!sh_ipr) {
1476                sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1477                if (!sh_ipr)
1478                        return IRQ_NONE;
1479                sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1480                bank = 1;
1481        } else {
1482                sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1483                bank = 0;
1484        }
1485
1486        do {
1487                u32 slot;
1488                u32 channel;
1489
1490                slot = __ffs(sh_ipr);
1491                sh_ipr &= ~(BIT(slot));
1492
1493                if (sh_ier & BIT(slot)) {
1494                        channel = (bank << 5) | slot;
1495                        /* Clear the corresponding IPR bits */
1496                        edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1497                        edma_completion_handler(&ecc->slave_chans[channel]);
1498                }
1499        } while (sh_ipr);
1500
1501        edma_shadow0_write(ecc, SH_IEVAL, 1);
1502        return IRQ_HANDLED;
1503}
1504
1505static void edma_error_handler(struct edma_chan *echan)
1506{
1507        struct edma_cc *ecc = echan->ecc;
1508        struct device *dev = echan->vchan.chan.device->dev;
1509        struct edmacc_param p;
1510        int err;
1511
1512        if (!echan->edesc)
1513                return;
1514
1515        spin_lock(&echan->vchan.lock);
1516
1517        err = edma_read_slot(ecc, echan->slot[0], &p);
1518
1519        /*
1520         * Issue later based on missed flag which will be sure
1521         * to happen as:
1522         * (1) we finished transmitting an intermediate slot and
1523         *     edma_execute is coming up.
1524         * (2) or we finished current transfer and issue will
1525         *     call edma_execute.
1526         *
1527         * Important note: issuing can be dangerous here and
1528         * lead to some nasty recursion when we are in a NULL
1529         * slot. So we avoid doing so and set the missed flag.
1530         */
1531        if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
1532                dev_dbg(dev, "Error on null slot, setting miss\n");
1533                echan->missed = 1;
1534        } else {
1535                /*
1536                 * The slot is already programmed but the event got
1537                 * missed, so its safe to issue it here.
1538                 */
1539                dev_dbg(dev, "Missed event, TRIGGERING\n");
1540                edma_clean_channel(echan);
1541                edma_stop(echan);
1542                edma_start(echan);
1543                edma_trigger_channel(echan);
1544        }
1545        spin_unlock(&echan->vchan.lock);
1546}
1547
1548static inline bool edma_error_pending(struct edma_cc *ecc)
1549{
1550        if (edma_read_array(ecc, EDMA_EMR, 0) ||
1551            edma_read_array(ecc, EDMA_EMR, 1) ||
1552            edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1553                return true;
1554
1555        return false;
1556}
1557
1558/* eDMA error interrupt handler */
1559static irqreturn_t dma_ccerr_handler(int irq, void *data)
1560{
1561        struct edma_cc *ecc = data;
1562        int i, j;
1563        int ctlr;
1564        unsigned int cnt = 0;
1565        unsigned int val;
1566
1567        ctlr = ecc->id;
1568        if (ctlr < 0)
1569                return IRQ_NONE;
1570
1571        dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1572
1573        if (!edma_error_pending(ecc)) {
1574                /*
1575                 * The registers indicate no pending error event but the irq
1576                 * handler has been called.
1577                 * Ask eDMA to re-evaluate the error registers.
1578                 */
1579                dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
1580                        __func__);
1581                edma_write(ecc, EDMA_EEVAL, 1);
1582                return IRQ_NONE;
1583        }
1584
1585        while (1) {
1586                /* Event missed register(s) */
1587                for (j = 0; j < 2; j++) {
1588                        unsigned long emr;
1589
1590                        val = edma_read_array(ecc, EDMA_EMR, j);
1591                        if (!val)
1592                                continue;
1593
1594                        dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
1595                        emr = val;
1596                        for (i = find_next_bit(&emr, 32, 0); i < 32;
1597                             i = find_next_bit(&emr, 32, i + 1)) {
1598                                int k = (j << 5) + i;
1599
1600                                /* Clear the corresponding EMR bits */
1601                                edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
1602                                /* Clear any SER */
1603                                edma_shadow0_write_array(ecc, SH_SECR, j,
1604                                                         BIT(i));
1605                                edma_error_handler(&ecc->slave_chans[k]);
1606                        }
1607                }
1608
1609                val = edma_read(ecc, EDMA_QEMR);
1610                if (val) {
1611                        dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
1612                        /* Not reported, just clear the interrupt reason. */
1613                        edma_write(ecc, EDMA_QEMCR, val);
1614                        edma_shadow0_write(ecc, SH_QSECR, val);
1615                }
1616
1617                val = edma_read(ecc, EDMA_CCERR);
1618                if (val) {
1619                        dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
1620                        /* Not reported, just clear the interrupt reason. */
1621                        edma_write(ecc, EDMA_CCERRCLR, val);
1622                }
1623
1624                if (!edma_error_pending(ecc))
1625                        break;
1626                cnt++;
1627                if (cnt > 10)
1628                        break;
1629        }
1630        edma_write(ecc, EDMA_EEVAL, 1);
1631        return IRQ_HANDLED;
1632}
1633
1634/* Alloc channel resources */
1635static int edma_alloc_chan_resources(struct dma_chan *chan)
1636{
1637        struct edma_chan *echan = to_edma_chan(chan);
1638        struct edma_cc *ecc = echan->ecc;
1639        struct device *dev = ecc->dev;
1640        enum dma_event_q eventq_no = EVENTQ_DEFAULT;
1641        int ret;
1642
1643        if (echan->tc) {
1644                eventq_no = echan->tc->id;
1645        } else if (ecc->tc_list) {
1646                /* memcpy channel */
1647                echan->tc = &ecc->tc_list[ecc->info->default_queue];
1648                eventq_no = echan->tc->id;
1649        }
1650
1651        ret = edma_alloc_channel(echan, eventq_no);
1652        if (ret)
1653                return ret;
1654
1655        echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
1656        if (echan->slot[0] < 0) {
1657                dev_err(dev, "Entry slot allocation failed for channel %u\n",
1658                        EDMA_CHAN_SLOT(echan->ch_num));
1659                ret = echan->slot[0];
1660                goto err_slot;
1661        }
1662
1663        /* Set up channel -> slot mapping for the entry slot */
1664        edma_set_chmap(echan, echan->slot[0]);
1665        echan->alloced = true;
1666
1667        dev_dbg(dev, "Got eDMA channel %d for virt channel %d (%s trigger)\n",
1668                EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id,
1669                echan->hw_triggered ? "HW" : "SW");
1670
1671        return 0;
1672
1673err_slot:
1674        edma_free_channel(echan);
1675        return ret;
1676}
1677
1678/* Free channel resources */
1679static void edma_free_chan_resources(struct dma_chan *chan)
1680{
1681        struct edma_chan *echan = to_edma_chan(chan);
1682        struct device *dev = echan->ecc->dev;
1683        int i;
1684
1685        /* Terminate transfers */
1686        edma_stop(echan);
1687
1688        vchan_free_chan_resources(&echan->vchan);
1689
1690        /* Free EDMA PaRAM slots */
1691        for (i = 0; i < EDMA_MAX_SLOTS; i++) {
1692                if (echan->slot[i] >= 0) {
1693                        edma_free_slot(echan->ecc, echan->slot[i]);
1694                        echan->slot[i] = -1;
1695                }
1696        }
1697
1698        /* Set entry slot to the dummy slot */
1699        edma_set_chmap(echan, echan->ecc->dummy_slot);
1700
1701        /* Free EDMA channel */
1702        if (echan->alloced) {
1703                edma_free_channel(echan);
1704                echan->alloced = false;
1705        }
1706
1707        echan->tc = NULL;
1708        echan->hw_triggered = false;
1709
1710        dev_dbg(dev, "Free eDMA channel %d for virt channel %d\n",
1711                EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id);
1712}
1713
1714/* Send pending descriptor to hardware */
1715static void edma_issue_pending(struct dma_chan *chan)
1716{
1717        struct edma_chan *echan = to_edma_chan(chan);
1718        unsigned long flags;
1719
1720        spin_lock_irqsave(&echan->vchan.lock, flags);
1721        if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
1722                edma_execute(echan);
1723        spin_unlock_irqrestore(&echan->vchan.lock, flags);
1724}
1725
1726/*
1727 * This limit exists to avoid a possible infinite loop when waiting for proof
1728 * that a particular transfer is completed. This limit can be hit if there
1729 * are large bursts to/from slow devices or the CPU is never able to catch
1730 * the DMA hardware idle. On an AM335x transfering 48 bytes from the UART
1731 * RX-FIFO, as many as 55 loops have been seen.
1732 */
1733#define EDMA_MAX_TR_WAIT_LOOPS 1000
1734
1735static u32 edma_residue(struct edma_desc *edesc)
1736{
1737        bool dst = edesc->direction == DMA_DEV_TO_MEM;
1738        int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
1739        struct edma_chan *echan = edesc->echan;
1740        struct edma_pset *pset = edesc->pset;
1741        dma_addr_t done, pos, pos_old;
1742        int channel = EDMA_CHAN_SLOT(echan->ch_num);
1743        int idx = EDMA_REG_ARRAY_INDEX(channel);
1744        int ch_bit = EDMA_CHANNEL_BIT(channel);
1745        int event_reg;
1746        int i;
1747
1748        /*
1749         * We always read the dst/src position from the first RamPar
1750         * pset. That's the one which is active now.
1751         */
1752        pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1753
1754        /*
1755         * "pos" may represent a transfer request that is still being
1756         * processed by the EDMACC or EDMATC. We will busy wait until
1757         * any one of the situations occurs:
1758         *   1. while and event is pending for the channel
1759         *   2. a position updated
1760         *   3. we hit the loop limit
1761         */
1762        if (is_slave_direction(edesc->direction))
1763                event_reg = SH_ER;
1764        else
1765                event_reg = SH_ESR;
1766
1767        pos_old = pos;
1768        while (edma_shadow0_read_array(echan->ecc, event_reg, idx) & ch_bit) {
1769                pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1770                if (pos != pos_old)
1771                        break;
1772
1773                if (!--loop_count) {
1774                        dev_dbg_ratelimited(echan->vchan.chan.device->dev,
1775                                "%s: timeout waiting for PaRAM update\n",
1776                                __func__);
1777                        break;
1778                }
1779
1780                cpu_relax();
1781        }
1782
1783        /*
1784         * Cyclic is simple. Just subtract pset[0].addr from pos.
1785         *
1786         * We never update edesc->residue in the cyclic case, so we
1787         * can tell the remaining room to the end of the circular
1788         * buffer.
1789         */
1790        if (edesc->cyclic) {
1791                done = pos - pset->addr;
1792                edesc->residue_stat = edesc->residue - done;
1793                return edesc->residue_stat;
1794        }
1795
1796        /*
1797         * If the position is 0, then EDMA loaded the closing dummy slot, the
1798         * transfer is completed
1799         */
1800        if (!pos)
1801                return 0;
1802        /*
1803         * For SG operation we catch up with the last processed
1804         * status.
1805         */
1806        pset += edesc->processed_stat;
1807
1808        for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
1809                /*
1810                 * If we are inside this pset address range, we know
1811                 * this is the active one. Get the current delta and
1812                 * stop walking the psets.
1813                 */
1814                if (pos >= pset->addr && pos < pset->addr + pset->len)
1815                        return edesc->residue_stat - (pos - pset->addr);
1816
1817                /* Otherwise mark it done and update residue_stat. */
1818                edesc->processed_stat++;
1819                edesc->residue_stat -= pset->len;
1820        }
1821        return edesc->residue_stat;
1822}
1823
1824/* Check request completion status */
1825static enum dma_status edma_tx_status(struct dma_chan *chan,
1826                                      dma_cookie_t cookie,
1827                                      struct dma_tx_state *txstate)
1828{
1829        struct edma_chan *echan = to_edma_chan(chan);
1830        struct dma_tx_state txstate_tmp;
1831        enum dma_status ret;
1832        unsigned long flags;
1833
1834        ret = dma_cookie_status(chan, cookie, txstate);
1835
1836        if (ret == DMA_COMPLETE)
1837                return ret;
1838
1839        /* Provide a dummy dma_tx_state for completion checking */
1840        if (!txstate)
1841                txstate = &txstate_tmp;
1842
1843        spin_lock_irqsave(&echan->vchan.lock, flags);
1844        if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {
1845                txstate->residue = edma_residue(echan->edesc);
1846        } else {
1847                struct virt_dma_desc *vdesc = vchan_find_desc(&echan->vchan,
1848                                                              cookie);
1849
1850                if (vdesc)
1851                        txstate->residue = to_edma_desc(&vdesc->tx)->residue;
1852                else
1853                        txstate->residue = 0;
1854        }
1855
1856        /*
1857         * Mark the cookie completed if the residue is 0 for non cyclic
1858         * transfers
1859         */
1860        if (ret != DMA_COMPLETE && !txstate->residue &&
1861            echan->edesc && echan->edesc->polled &&
1862            echan->edesc->vdesc.tx.cookie == cookie) {
1863                edma_stop(echan);
1864                vchan_cookie_complete(&echan->edesc->vdesc);
1865                echan->edesc = NULL;
1866                edma_execute(echan);
1867                ret = DMA_COMPLETE;
1868        }
1869
1870        spin_unlock_irqrestore(&echan->vchan.lock, flags);
1871
1872        return ret;
1873}
1874
1875static bool edma_is_memcpy_channel(int ch_num, s32 *memcpy_channels)
1876{
1877        if (!memcpy_channels)
1878                return false;
1879        while (*memcpy_channels != -1) {
1880                if (*memcpy_channels == ch_num)
1881                        return true;
1882                memcpy_channels++;
1883        }
1884        return false;
1885}
1886
1887#define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1888                                 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1889                                 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
1890                                 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1891
1892static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
1893{
1894        struct dma_device *s_ddev = &ecc->dma_slave;
1895        struct dma_device *m_ddev = NULL;
1896        s32 *memcpy_channels = ecc->info->memcpy_channels;
1897        int i, j;
1898
1899        dma_cap_zero(s_ddev->cap_mask);
1900        dma_cap_set(DMA_SLAVE, s_ddev->cap_mask);
1901        dma_cap_set(DMA_CYCLIC, s_ddev->cap_mask);
1902        if (ecc->legacy_mode && !memcpy_channels) {
1903                dev_warn(ecc->dev,
1904                         "Legacy memcpy is enabled, things might not work\n");
1905
1906                dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask);
1907                s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1908                s_ddev->directions = BIT(DMA_MEM_TO_MEM);
1909        }
1910
1911        s_ddev->device_prep_slave_sg = edma_prep_slave_sg;
1912        s_ddev->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1913        s_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1914        s_ddev->device_free_chan_resources = edma_free_chan_resources;
1915        s_ddev->device_issue_pending = edma_issue_pending;
1916        s_ddev->device_tx_status = edma_tx_status;
1917        s_ddev->device_config = edma_slave_config;
1918        s_ddev->device_pause = edma_dma_pause;
1919        s_ddev->device_resume = edma_dma_resume;
1920        s_ddev->device_terminate_all = edma_terminate_all;
1921        s_ddev->device_synchronize = edma_synchronize;
1922
1923        s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1924        s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1925        s_ddev->directions |= (BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV));
1926        s_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1927        s_ddev->max_burst = SZ_32K - 1; /* CIDX: 16bit signed */
1928
1929        s_ddev->dev = ecc->dev;
1930        INIT_LIST_HEAD(&s_ddev->channels);
1931
1932        if (memcpy_channels) {
1933                m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
1934                if (!m_ddev) {
1935                        dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
1936                        memcpy_channels = NULL;
1937                        goto ch_setup;
1938                }
1939                ecc->dma_memcpy = m_ddev;
1940
1941                dma_cap_zero(m_ddev->cap_mask);
1942                dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask);
1943
1944                m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1945                m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1946                m_ddev->device_free_chan_resources = edma_free_chan_resources;
1947                m_ddev->device_issue_pending = edma_issue_pending;
1948                m_ddev->device_tx_status = edma_tx_status;
1949                m_ddev->device_config = edma_slave_config;
1950                m_ddev->device_pause = edma_dma_pause;
1951                m_ddev->device_resume = edma_dma_resume;
1952                m_ddev->device_terminate_all = edma_terminate_all;
1953                m_ddev->device_synchronize = edma_synchronize;
1954
1955                m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1956                m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1957                m_ddev->directions = BIT(DMA_MEM_TO_MEM);
1958                m_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1959
1960                m_ddev->dev = ecc->dev;
1961                INIT_LIST_HEAD(&m_ddev->channels);
1962        } else if (!ecc->legacy_mode) {
1963                dev_info(ecc->dev, "memcpy is disabled\n");
1964        }
1965
1966ch_setup:
1967        for (i = 0; i < ecc->num_channels; i++) {
1968                struct edma_chan *echan = &ecc->slave_chans[i];
1969                echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
1970                echan->ecc = ecc;
1971                echan->vchan.desc_free = edma_desc_free;
1972
1973                if (m_ddev && edma_is_memcpy_channel(i, memcpy_channels))
1974                        vchan_init(&echan->vchan, m_ddev);
1975                else
1976                        vchan_init(&echan->vchan, s_ddev);
1977
1978                INIT_LIST_HEAD(&echan->node);
1979                for (j = 0; j < EDMA_MAX_SLOTS; j++)
1980                        echan->slot[j] = -1;
1981        }
1982}
1983
1984static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
1985                              struct edma_cc *ecc)
1986{
1987        int i;
1988        u32 value, cccfg;
1989        s8 (*queue_priority_map)[2];
1990
1991        /* Decode the eDMA3 configuration from CCCFG register */
1992        cccfg = edma_read(ecc, EDMA_CCCFG);
1993
1994        value = GET_NUM_REGN(cccfg);
1995        ecc->num_region = BIT(value);
1996
1997        value = GET_NUM_DMACH(cccfg);
1998        ecc->num_channels = BIT(value + 1);
1999
2000        value = GET_NUM_QDMACH(cccfg);
2001        ecc->num_qchannels = value * 2;
2002
2003        value = GET_NUM_PAENTRY(cccfg);
2004        ecc->num_slots = BIT(value + 4);
2005
2006        value = GET_NUM_EVQUE(cccfg);
2007        ecc->num_tc = value + 1;
2008
2009        ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
2010
2011        dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
2012        dev_dbg(dev, "num_region: %u\n", ecc->num_region);
2013        dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
2014        dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
2015        dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
2016        dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
2017        dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
2018
2019        /* Nothing need to be done if queue priority is provided */
2020        if (pdata->queue_priority_mapping)
2021                return 0;
2022
2023        /*
2024         * Configure TC/queue priority as follows:
2025         * Q0 - priority 0
2026         * Q1 - priority 1
2027         * Q2 - priority 2
2028         * ...
2029         * The meaning of priority numbers: 0 highest priority, 7 lowest
2030         * priority. So Q0 is the highest priority queue and the last queue has
2031         * the lowest priority.
2032         */
2033        queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
2034                                          GFP_KERNEL);
2035        if (!queue_priority_map)
2036                return -ENOMEM;
2037
2038        for (i = 0; i < ecc->num_tc; i++) {
2039                queue_priority_map[i][0] = i;
2040                queue_priority_map[i][1] = i;
2041        }
2042        queue_priority_map[i][0] = -1;
2043        queue_priority_map[i][1] = -1;
2044
2045        pdata->queue_priority_mapping = queue_priority_map;
2046        /* Default queue has the lowest priority */
2047        pdata->default_queue = i - 1;
2048
2049        return 0;
2050}
2051
2052#if IS_ENABLED(CONFIG_OF)
2053static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
2054                               size_t sz)
2055{
2056        const char pname[] = "ti,edma-xbar-event-map";
2057        struct resource res;
2058        void __iomem *xbar;
2059        s16 (*xbar_chans)[2];
2060        size_t nelm = sz / sizeof(s16);
2061        u32 shift, offset, mux;
2062        int ret, i;
2063
2064        xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
2065        if (!xbar_chans)
2066                return -ENOMEM;
2067
2068        ret = of_address_to_resource(dev->of_node, 1, &res);
2069        if (ret)
2070                return -ENOMEM;
2071
2072        xbar = devm_ioremap(dev, res.start, resource_size(&res));
2073        if (!xbar)
2074                return -ENOMEM;
2075
2076        ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
2077                                         nelm);
2078        if (ret)
2079                return -EIO;
2080
2081        /* Invalidate last entry for the other user of this mess */
2082        nelm >>= 1;
2083        xbar_chans[nelm][0] = -1;
2084        xbar_chans[nelm][1] = -1;
2085
2086        for (i = 0; i < nelm; i++) {
2087                shift = (xbar_chans[i][1] & 0x03) << 3;
2088                offset = xbar_chans[i][1] & 0xfffffffc;
2089                mux = readl(xbar + offset);
2090                mux &= ~(0xff << shift);
2091                mux |= xbar_chans[i][0] << shift;
2092                writel(mux, (xbar + offset));
2093        }
2094
2095        pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
2096        return 0;
2097}
2098
2099static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2100                                                     bool legacy_mode)
2101{
2102        struct edma_soc_info *info;
2103        struct property *prop;
2104        int sz, ret;
2105
2106        info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
2107        if (!info)
2108                return ERR_PTR(-ENOMEM);
2109
2110        if (legacy_mode) {
2111                prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map",
2112                                        &sz);
2113                if (prop) {
2114                        ret = edma_xbar_event_map(dev, info, sz);
2115                        if (ret)
2116                                return ERR_PTR(ret);
2117                }
2118                return info;
2119        }
2120
2121        /* Get the list of channels allocated to be used for memcpy */
2122        prop = of_find_property(dev->of_node, "ti,edma-memcpy-channels", &sz);
2123        if (prop) {
2124                const char pname[] = "ti,edma-memcpy-channels";
2125                size_t nelm = sz / sizeof(s32);
2126                s32 *memcpy_ch;
2127
2128                memcpy_ch = devm_kcalloc(dev, nelm + 1, sizeof(s32),
2129                                         GFP_KERNEL);
2130                if (!memcpy_ch)
2131                        return ERR_PTR(-ENOMEM);
2132
2133                ret = of_property_read_u32_array(dev->of_node, pname,
2134                                                 (u32 *)memcpy_ch, nelm);
2135                if (ret)
2136                        return ERR_PTR(ret);
2137
2138                memcpy_ch[nelm] = -1;
2139                info->memcpy_channels = memcpy_ch;
2140        }
2141
2142        prop = of_find_property(dev->of_node, "ti,edma-reserved-slot-ranges",
2143                                &sz);
2144        if (prop) {
2145                const char pname[] = "ti,edma-reserved-slot-ranges";
2146                u32 (*tmp)[2];
2147                s16 (*rsv_slots)[2];
2148                size_t nelm = sz / sizeof(*tmp);
2149                struct edma_rsv_info *rsv_info;
2150                int i;
2151
2152                if (!nelm)
2153                        return info;
2154
2155                tmp = kcalloc(nelm, sizeof(*tmp), GFP_KERNEL);
2156                if (!tmp)
2157                        return ERR_PTR(-ENOMEM);
2158
2159                rsv_info = devm_kzalloc(dev, sizeof(*rsv_info), GFP_KERNEL);
2160                if (!rsv_info) {
2161                        kfree(tmp);
2162                        return ERR_PTR(-ENOMEM);
2163                }
2164
2165                rsv_slots = devm_kcalloc(dev, nelm + 1, sizeof(*rsv_slots),
2166                                         GFP_KERNEL);
2167                if (!rsv_slots) {
2168                        kfree(tmp);
2169                        return ERR_PTR(-ENOMEM);
2170                }
2171
2172                ret = of_property_read_u32_array(dev->of_node, pname,
2173                                                 (u32 *)tmp, nelm * 2);
2174                if (ret) {
2175                        kfree(tmp);
2176                        return ERR_PTR(ret);
2177                }
2178
2179                for (i = 0; i < nelm; i++) {
2180                        rsv_slots[i][0] = tmp[i][0];
2181                        rsv_slots[i][1] = tmp[i][1];
2182                }
2183                rsv_slots[nelm][0] = -1;
2184                rsv_slots[nelm][1] = -1;
2185
2186                info->rsv = rsv_info;
2187                info->rsv->rsv_slots = (const s16 (*)[2])rsv_slots;
2188
2189                kfree(tmp);
2190        }
2191
2192        return info;
2193}
2194
2195static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2196                                      struct of_dma *ofdma)
2197{
2198        struct edma_cc *ecc = ofdma->of_dma_data;
2199        struct dma_chan *chan = NULL;
2200        struct edma_chan *echan;
2201        int i;
2202
2203        if (!ecc || dma_spec->args_count < 1)
2204                return NULL;
2205
2206        for (i = 0; i < ecc->num_channels; i++) {
2207                echan = &ecc->slave_chans[i];
2208                if (echan->ch_num == dma_spec->args[0]) {
2209                        chan = &echan->vchan.chan;
2210                        break;
2211                }
2212        }
2213
2214        if (!chan)
2215                return NULL;
2216
2217        if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
2218                goto out;
2219
2220        if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
2221            dma_spec->args[1] < echan->ecc->num_tc) {
2222                echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
2223                goto out;
2224        }
2225
2226        return NULL;
2227out:
2228        /* The channel is going to be used as HW synchronized */
2229        echan->hw_triggered = true;
2230        return dma_get_slave_channel(chan);
2231}
2232#else
2233static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2234                                                     bool legacy_mode)
2235{
2236        return ERR_PTR(-EINVAL);
2237}
2238
2239static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2240                                      struct of_dma *ofdma)
2241{
2242        return NULL;
2243}
2244#endif
2245
2246static bool edma_filter_fn(struct dma_chan *chan, void *param);
2247
2248static int edma_probe(struct platform_device *pdev)
2249{
2250        struct edma_soc_info    *info = pdev->dev.platform_data;
2251        s8                      (*queue_priority_mapping)[2];
2252        int                     i, off;
2253        const s16               (*rsv_slots)[2];
2254        const s16               (*xbar_chans)[2];
2255        int                     irq;
2256        char                    *irq_name;
2257        struct resource         *mem;
2258        struct device_node      *node = pdev->dev.of_node;
2259        struct device           *dev = &pdev->dev;
2260        struct edma_cc          *ecc;
2261        bool                    legacy_mode = true;
2262        int ret;
2263
2264        if (node) {
2265                const struct of_device_id *match;
2266
2267                match = of_match_node(edma_of_ids, node);
2268                if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
2269                        legacy_mode = false;
2270
2271                info = edma_setup_info_from_dt(dev, legacy_mode);
2272                if (IS_ERR(info)) {
2273                        dev_err(dev, "failed to get DT data\n");
2274                        return PTR_ERR(info);
2275                }
2276        }
2277
2278        if (!info)
2279                return -ENODEV;
2280
2281        pm_runtime_enable(dev);
2282        ret = pm_runtime_get_sync(dev);
2283        if (ret < 0) {
2284                dev_err(dev, "pm_runtime_get_sync() failed\n");
2285                return ret;
2286        }
2287
2288        ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
2289        if (ret)
2290                return ret;
2291
2292        ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
2293        if (!ecc)
2294                return -ENOMEM;
2295
2296        ecc->dev = dev;
2297        ecc->id = pdev->id;
2298        ecc->legacy_mode = legacy_mode;
2299        /* When booting with DT the pdev->id is -1 */
2300        if (ecc->id < 0)
2301                ecc->id = 0;
2302
2303        mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
2304        if (!mem) {
2305                dev_dbg(dev, "mem resource not found, using index 0\n");
2306                mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2307                if (!mem) {
2308                        dev_err(dev, "no mem resource?\n");
2309                        return -ENODEV;
2310                }
2311        }
2312        ecc->base = devm_ioremap_resource(dev, mem);
2313        if (IS_ERR(ecc->base))
2314                return PTR_ERR(ecc->base);
2315
2316        platform_set_drvdata(pdev, ecc);
2317
2318        /* Get eDMA3 configuration from IP */
2319        ret = edma_setup_from_hw(dev, info, ecc);
2320        if (ret)
2321                return ret;
2322
2323        /* Allocate memory based on the information we got from the IP */
2324        ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2325                                        sizeof(*ecc->slave_chans), GFP_KERNEL);
2326        if (!ecc->slave_chans)
2327                return -ENOMEM;
2328
2329        ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
2330                                       sizeof(unsigned long), GFP_KERNEL);
2331        if (!ecc->slot_inuse)
2332                return -ENOMEM;
2333
2334        ecc->default_queue = info->default_queue;
2335
2336        if (info->rsv) {
2337                /* Set the reserved slots in inuse list */
2338                rsv_slots = info->rsv->rsv_slots;
2339                if (rsv_slots) {
2340                        for (i = 0; rsv_slots[i][0] != -1; i++)
2341                                bitmap_set(ecc->slot_inuse, rsv_slots[i][0],
2342                                           rsv_slots[i][1]);
2343                }
2344        }
2345
2346        for (i = 0; i < ecc->num_slots; i++) {
2347                /* Reset only unused - not reserved - paRAM slots */
2348                if (!test_bit(i, ecc->slot_inuse))
2349                        edma_write_slot(ecc, i, &dummy_paramset);
2350        }
2351
2352        /* Clear the xbar mapped channels in unused list */
2353        xbar_chans = info->xbar_chans;
2354        if (xbar_chans) {
2355                for (i = 0; xbar_chans[i][1] != -1; i++) {
2356                        off = xbar_chans[i][1];
2357                }
2358        }
2359
2360        irq = platform_get_irq_byname(pdev, "edma3_ccint");
2361        if (irq < 0 && node)
2362                irq = irq_of_parse_and_map(node, 0);
2363
2364        if (irq >= 0) {
2365                irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
2366                                          dev_name(dev));
2367                ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
2368                                       ecc);
2369                if (ret) {
2370                        dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
2371                        return ret;
2372                }
2373                ecc->ccint = irq;
2374        }
2375
2376        irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
2377        if (irq < 0 && node)
2378                irq = irq_of_parse_and_map(node, 2);
2379
2380        if (irq >= 0) {
2381                irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
2382                                          dev_name(dev));
2383                ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
2384                                       ecc);
2385                if (ret) {
2386                        dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
2387                        return ret;
2388                }
2389                ecc->ccerrint = irq;
2390        }
2391
2392        ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
2393        if (ecc->dummy_slot < 0) {
2394                dev_err(dev, "Can't allocate PaRAM dummy slot\n");
2395                return ecc->dummy_slot;
2396        }
2397
2398        queue_priority_mapping = info->queue_priority_mapping;
2399
2400        if (!ecc->legacy_mode) {
2401                int lowest_priority = 0;
2402                struct of_phandle_args tc_args;
2403
2404                ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
2405                                            sizeof(*ecc->tc_list), GFP_KERNEL);
2406                if (!ecc->tc_list)
2407                        return -ENOMEM;
2408
2409                for (i = 0;; i++) {
2410                        ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs",
2411                                                               1, i, &tc_args);
2412                        if (ret || i == ecc->num_tc)
2413                                break;
2414
2415                        ecc->tc_list[i].node = tc_args.np;
2416                        ecc->tc_list[i].id = i;
2417                        queue_priority_mapping[i][1] = tc_args.args[0];
2418                        if (queue_priority_mapping[i][1] > lowest_priority) {
2419                                lowest_priority = queue_priority_mapping[i][1];
2420                                info->default_queue = i;
2421                        }
2422                }
2423        }
2424
2425        /* Event queue priority mapping */
2426        for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2427                edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2428                                              queue_priority_mapping[i][1]);
2429
2430        edma_write_array2(ecc, EDMA_DRAE, 0, 0, 0x0);
2431        edma_write_array2(ecc, EDMA_DRAE, 0, 1, 0x0);
2432        edma_write_array(ecc, EDMA_QRAE, 0, 0x0);
2433
2434        ecc->info = info;
2435
2436        /* Init the dma device and channels */
2437        edma_dma_init(ecc, legacy_mode);
2438
2439        for (i = 0; i < ecc->num_channels; i++) {
2440                /* Assign all channels to the default queue */
2441                edma_assign_channel_eventq(&ecc->slave_chans[i],
2442                                           info->default_queue);
2443                /* Set entry slot to the dummy slot */
2444                edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
2445        }
2446
2447        ecc->dma_slave.filter.map = info->slave_map;
2448        ecc->dma_slave.filter.mapcnt = info->slavecnt;
2449        ecc->dma_slave.filter.fn = edma_filter_fn;
2450
2451        ret = dma_async_device_register(&ecc->dma_slave);
2452        if (ret) {
2453                dev_err(dev, "slave ddev registration failed (%d)\n", ret);
2454                goto err_reg1;
2455        }
2456
2457        if (ecc->dma_memcpy) {
2458                ret = dma_async_device_register(ecc->dma_memcpy);
2459                if (ret) {
2460                        dev_err(dev, "memcpy ddev registration failed (%d)\n",
2461                                ret);
2462                        dma_async_device_unregister(&ecc->dma_slave);
2463                        goto err_reg1;
2464                }
2465        }
2466
2467        if (node)
2468                of_dma_controller_register(node, of_edma_xlate, ecc);
2469
2470        dev_info(dev, "TI EDMA DMA engine driver\n");
2471
2472        return 0;
2473
2474err_reg1:
2475        edma_free_slot(ecc, ecc->dummy_slot);
2476        return ret;
2477}
2478
2479static void edma_cleanupp_vchan(struct dma_device *dmadev)
2480{
2481        struct edma_chan *echan, *_echan;
2482
2483        list_for_each_entry_safe(echan, _echan,
2484                        &dmadev->channels, vchan.chan.device_node) {
2485                list_del(&echan->vchan.chan.device_node);
2486                tasklet_kill(&echan->vchan.task);
2487        }
2488}
2489
2490static int edma_remove(struct platform_device *pdev)
2491{
2492        struct device *dev = &pdev->dev;
2493        struct edma_cc *ecc = dev_get_drvdata(dev);
2494
2495        devm_free_irq(dev, ecc->ccint, ecc);
2496        devm_free_irq(dev, ecc->ccerrint, ecc);
2497
2498        edma_cleanupp_vchan(&ecc->dma_slave);
2499
2500        if (dev->of_node)
2501                of_dma_controller_free(dev->of_node);
2502        dma_async_device_unregister(&ecc->dma_slave);
2503        if (ecc->dma_memcpy)
2504                dma_async_device_unregister(ecc->dma_memcpy);
2505        edma_free_slot(ecc, ecc->dummy_slot);
2506
2507        return 0;
2508}
2509
2510#ifdef CONFIG_PM_SLEEP
2511static int edma_pm_suspend(struct device *dev)
2512{
2513        struct edma_cc *ecc = dev_get_drvdata(dev);
2514        struct edma_chan *echan = ecc->slave_chans;
2515        int i;
2516
2517        for (i = 0; i < ecc->num_channels; i++) {
2518                if (echan[i].alloced)
2519                        edma_setup_interrupt(&echan[i], false);
2520        }
2521
2522        return 0;
2523}
2524
2525static int edma_pm_resume(struct device *dev)
2526{
2527        struct edma_cc *ecc = dev_get_drvdata(dev);
2528        struct edma_chan *echan = ecc->slave_chans;
2529        int i;
2530        s8 (*queue_priority_mapping)[2];
2531
2532        /* re initialize dummy slot to dummy param set */
2533        edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
2534
2535        queue_priority_mapping = ecc->info->queue_priority_mapping;
2536
2537        /* Event queue priority mapping */
2538        for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2539                edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2540                                              queue_priority_mapping[i][1]);
2541
2542        for (i = 0; i < ecc->num_channels; i++) {
2543                if (echan[i].alloced) {
2544                        /* ensure access through shadow region 0 */
2545                        edma_or_array2(ecc, EDMA_DRAE, 0,
2546                                       EDMA_REG_ARRAY_INDEX(i),
2547                                       EDMA_CHANNEL_BIT(i));
2548
2549                        edma_setup_interrupt(&echan[i], true);
2550
2551                        /* Set up channel -> slot mapping for the entry slot */
2552                        edma_set_chmap(&echan[i], echan[i].slot[0]);
2553                }
2554        }
2555
2556        return 0;
2557}
2558#endif
2559
2560static const struct dev_pm_ops edma_pm_ops = {
2561        SET_LATE_SYSTEM_SLEEP_PM_OPS(edma_pm_suspend, edma_pm_resume)
2562};
2563
2564static struct platform_driver edma_driver = {
2565        .probe          = edma_probe,
2566        .remove         = edma_remove,
2567        .driver = {
2568                .name   = "edma",
2569                .pm     = &edma_pm_ops,
2570                .of_match_table = edma_of_ids,
2571        },
2572};
2573
2574static int edma_tptc_probe(struct platform_device *pdev)
2575{
2576        pm_runtime_enable(&pdev->dev);
2577        return pm_runtime_get_sync(&pdev->dev);
2578}
2579
2580static struct platform_driver edma_tptc_driver = {
2581        .probe          = edma_tptc_probe,
2582        .driver = {
2583                .name   = "edma3-tptc",
2584                .of_match_table = edma_tptc_of_ids,
2585        },
2586};
2587
2588static bool edma_filter_fn(struct dma_chan *chan, void *param)
2589{
2590        bool match = false;
2591
2592        if (chan->device->dev->driver == &edma_driver.driver) {
2593                struct edma_chan *echan = to_edma_chan(chan);
2594                unsigned ch_req = *(unsigned *)param;
2595                if (ch_req == echan->ch_num) {
2596                        /* The channel is going to be used as HW synchronized */
2597                        echan->hw_triggered = true;
2598                        match = true;
2599                }
2600        }
2601        return match;
2602}
2603
2604static int edma_init(void)
2605{
2606        int ret;
2607
2608        ret = platform_driver_register(&edma_tptc_driver);
2609        if (ret)
2610                return ret;
2611
2612        return platform_driver_register(&edma_driver);
2613}
2614subsys_initcall(edma_init);
2615
2616static void __exit edma_exit(void)
2617{
2618        platform_driver_unregister(&edma_driver);
2619        platform_driver_unregister(&edma_tptc_driver);
2620}
2621module_exit(edma_exit);
2622
2623MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
2624MODULE_DESCRIPTION("TI EDMA DMA engine driver");
2625MODULE_LICENSE("GPL v2");
2626