linux/drivers/dma/sprd-dma.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2017 Spreadtrum Communications Inc.
   3 *
   4 * SPDX-License-Identifier: GPL-2.0
   5 */
   6
   7#include <linux/clk.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/dma/sprd-dma.h>
  10#include <linux/errno.h>
  11#include <linux/init.h>
  12#include <linux/interrupt.h>
  13#include <linux/io.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/of.h>
  17#include <linux/of_dma.h>
  18#include <linux/of_device.h>
  19#include <linux/pm_runtime.h>
  20#include <linux/slab.h>
  21
  22#include "virt-dma.h"
  23
  24#define SPRD_DMA_CHN_REG_OFFSET         0x1000
  25#define SPRD_DMA_CHN_REG_LENGTH         0x40
  26#define SPRD_DMA_MEMCPY_MIN_SIZE        64
  27
  28/* DMA global registers definition */
  29#define SPRD_DMA_GLB_PAUSE              0x0
  30#define SPRD_DMA_GLB_FRAG_WAIT          0x4
  31#define SPRD_DMA_GLB_REQ_PEND0_EN       0x8
  32#define SPRD_DMA_GLB_REQ_PEND1_EN       0xc
  33#define SPRD_DMA_GLB_INT_RAW_STS        0x10
  34#define SPRD_DMA_GLB_INT_MSK_STS        0x14
  35#define SPRD_DMA_GLB_REQ_STS            0x18
  36#define SPRD_DMA_GLB_CHN_EN_STS         0x1c
  37#define SPRD_DMA_GLB_DEBUG_STS          0x20
  38#define SPRD_DMA_GLB_ARB_SEL_STS        0x24
  39#define SPRD_DMA_GLB_2STAGE_GRP1        0x28
  40#define SPRD_DMA_GLB_2STAGE_GRP2        0x2c
  41#define SPRD_DMA_GLB_REQ_UID(uid)       (0x4 * ((uid) - 1))
  42#define SPRD_DMA_GLB_REQ_UID_OFFSET     0x2000
  43
  44/* DMA channel registers definition */
  45#define SPRD_DMA_CHN_PAUSE              0x0
  46#define SPRD_DMA_CHN_REQ                0x4
  47#define SPRD_DMA_CHN_CFG                0x8
  48#define SPRD_DMA_CHN_INTC               0xc
  49#define SPRD_DMA_CHN_SRC_ADDR           0x10
  50#define SPRD_DMA_CHN_DES_ADDR           0x14
  51#define SPRD_DMA_CHN_FRG_LEN            0x18
  52#define SPRD_DMA_CHN_BLK_LEN            0x1c
  53#define SPRD_DMA_CHN_TRSC_LEN           0x20
  54#define SPRD_DMA_CHN_TRSF_STEP          0x24
  55#define SPRD_DMA_CHN_WARP_PTR           0x28
  56#define SPRD_DMA_CHN_WARP_TO            0x2c
  57#define SPRD_DMA_CHN_LLIST_PTR          0x30
  58#define SPRD_DMA_CHN_FRAG_STEP          0x34
  59#define SPRD_DMA_CHN_SRC_BLK_STEP       0x38
  60#define SPRD_DMA_CHN_DES_BLK_STEP       0x3c
  61
  62/* SPRD_DMA_GLB_2STAGE_GRP register definition */
  63#define SPRD_DMA_GLB_2STAGE_EN          BIT(24)
  64#define SPRD_DMA_GLB_CHN_INT_MASK       GENMASK(23, 20)
  65#define SPRD_DMA_GLB_DEST_INT           BIT(22)
  66#define SPRD_DMA_GLB_SRC_INT            BIT(20)
  67#define SPRD_DMA_GLB_LIST_DONE_TRG      BIT(19)
  68#define SPRD_DMA_GLB_TRANS_DONE_TRG     BIT(18)
  69#define SPRD_DMA_GLB_BLOCK_DONE_TRG     BIT(17)
  70#define SPRD_DMA_GLB_FRAG_DONE_TRG      BIT(16)
  71#define SPRD_DMA_GLB_TRG_OFFSET         16
  72#define SPRD_DMA_GLB_DEST_CHN_MASK      GENMASK(13, 8)
  73#define SPRD_DMA_GLB_DEST_CHN_OFFSET    8
  74#define SPRD_DMA_GLB_SRC_CHN_MASK       GENMASK(5, 0)
  75
  76/* SPRD_DMA_CHN_INTC register definition */
  77#define SPRD_DMA_INT_MASK               GENMASK(4, 0)
  78#define SPRD_DMA_INT_CLR_OFFSET         24
  79#define SPRD_DMA_FRAG_INT_EN            BIT(0)
  80#define SPRD_DMA_BLK_INT_EN             BIT(1)
  81#define SPRD_DMA_TRANS_INT_EN           BIT(2)
  82#define SPRD_DMA_LIST_INT_EN            BIT(3)
  83#define SPRD_DMA_CFG_ERR_INT_EN         BIT(4)
  84
  85/* SPRD_DMA_CHN_CFG register definition */
  86#define SPRD_DMA_CHN_EN                 BIT(0)
  87#define SPRD_DMA_LINKLIST_EN            BIT(4)
  88#define SPRD_DMA_WAIT_BDONE_OFFSET      24
  89#define SPRD_DMA_DONOT_WAIT_BDONE       1
  90
  91/* SPRD_DMA_CHN_REQ register definition */
  92#define SPRD_DMA_REQ_EN                 BIT(0)
  93
  94/* SPRD_DMA_CHN_PAUSE register definition */
  95#define SPRD_DMA_PAUSE_EN               BIT(0)
  96#define SPRD_DMA_PAUSE_STS              BIT(2)
  97#define SPRD_DMA_PAUSE_CNT              0x2000
  98
  99/* DMA_CHN_WARP_* register definition */
 100#define SPRD_DMA_HIGH_ADDR_MASK         GENMASK(31, 28)
 101#define SPRD_DMA_LOW_ADDR_MASK          GENMASK(31, 0)
 102#define SPRD_DMA_WRAP_ADDR_MASK         GENMASK(27, 0)
 103#define SPRD_DMA_HIGH_ADDR_OFFSET       4
 104
 105/* SPRD_DMA_CHN_INTC register definition */
 106#define SPRD_DMA_FRAG_INT_STS           BIT(16)
 107#define SPRD_DMA_BLK_INT_STS            BIT(17)
 108#define SPRD_DMA_TRSC_INT_STS           BIT(18)
 109#define SPRD_DMA_LIST_INT_STS           BIT(19)
 110#define SPRD_DMA_CFGERR_INT_STS         BIT(20)
 111#define SPRD_DMA_CHN_INT_STS                                    \
 112        (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS |         \
 113         SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS |        \
 114         SPRD_DMA_CFGERR_INT_STS)
 115
 116/* SPRD_DMA_CHN_FRG_LEN register definition */
 117#define SPRD_DMA_SRC_DATAWIDTH_OFFSET   30
 118#define SPRD_DMA_DES_DATAWIDTH_OFFSET   28
 119#define SPRD_DMA_SWT_MODE_OFFSET        26
 120#define SPRD_DMA_REQ_MODE_OFFSET        24
 121#define SPRD_DMA_REQ_MODE_MASK          GENMASK(1, 0)
 122#define SPRD_DMA_WRAP_SEL_DEST          BIT(23)
 123#define SPRD_DMA_WRAP_EN                BIT(22)
 124#define SPRD_DMA_FIX_SEL_OFFSET         21
 125#define SPRD_DMA_FIX_EN_OFFSET          20
 126#define SPRD_DMA_LLIST_END              BIT(19)
 127#define SPRD_DMA_FRG_LEN_MASK           GENMASK(16, 0)
 128
 129/* SPRD_DMA_CHN_BLK_LEN register definition */
 130#define SPRD_DMA_BLK_LEN_MASK           GENMASK(16, 0)
 131
 132/* SPRD_DMA_CHN_TRSC_LEN register definition */
 133#define SPRD_DMA_TRSC_LEN_MASK          GENMASK(27, 0)
 134
 135/* SPRD_DMA_CHN_TRSF_STEP register definition */
 136#define SPRD_DMA_DEST_TRSF_STEP_OFFSET  16
 137#define SPRD_DMA_SRC_TRSF_STEP_OFFSET   0
 138#define SPRD_DMA_TRSF_STEP_MASK         GENMASK(15, 0)
 139
 140/* SPRD DMA_SRC_BLK_STEP register definition */
 141#define SPRD_DMA_LLIST_HIGH_MASK        GENMASK(31, 28)
 142#define SPRD_DMA_LLIST_HIGH_SHIFT       28
 143
 144/* define DMA channel mode & trigger mode mask */
 145#define SPRD_DMA_CHN_MODE_MASK          GENMASK(7, 0)
 146#define SPRD_DMA_TRG_MODE_MASK          GENMASK(7, 0)
 147#define SPRD_DMA_INT_TYPE_MASK          GENMASK(7, 0)
 148
 149/* define the DMA transfer step type */
 150#define SPRD_DMA_NONE_STEP              0
 151#define SPRD_DMA_BYTE_STEP              1
 152#define SPRD_DMA_SHORT_STEP             2
 153#define SPRD_DMA_WORD_STEP              4
 154#define SPRD_DMA_DWORD_STEP             8
 155
 156#define SPRD_DMA_SOFTWARE_UID           0
 157
 158/* dma data width values */
 159enum sprd_dma_datawidth {
 160        SPRD_DMA_DATAWIDTH_1_BYTE,
 161        SPRD_DMA_DATAWIDTH_2_BYTES,
 162        SPRD_DMA_DATAWIDTH_4_BYTES,
 163        SPRD_DMA_DATAWIDTH_8_BYTES,
 164};
 165
 166/* dma channel hardware configuration */
 167struct sprd_dma_chn_hw {
 168        u32 pause;
 169        u32 req;
 170        u32 cfg;
 171        u32 intc;
 172        u32 src_addr;
 173        u32 des_addr;
 174        u32 frg_len;
 175        u32 blk_len;
 176        u32 trsc_len;
 177        u32 trsf_step;
 178        u32 wrap_ptr;
 179        u32 wrap_to;
 180        u32 llist_ptr;
 181        u32 frg_step;
 182        u32 src_blk_step;
 183        u32 des_blk_step;
 184};
 185
 186/* dma request description */
 187struct sprd_dma_desc {
 188        struct virt_dma_desc    vd;
 189        struct sprd_dma_chn_hw  chn_hw;
 190        enum dma_transfer_direction dir;
 191};
 192
 193/* dma channel description */
 194struct sprd_dma_chn {
 195        struct virt_dma_chan    vc;
 196        void __iomem            *chn_base;
 197        struct sprd_dma_linklist        linklist;
 198        struct dma_slave_config slave_cfg;
 199        u32                     chn_num;
 200        u32                     dev_id;
 201        enum sprd_dma_chn_mode  chn_mode;
 202        enum sprd_dma_trg_mode  trg_mode;
 203        enum sprd_dma_int_type  int_type;
 204        struct sprd_dma_desc    *cur_desc;
 205};
 206
 207/* SPRD dma device */
 208struct sprd_dma_dev {
 209        struct dma_device       dma_dev;
 210        void __iomem            *glb_base;
 211        struct clk              *clk;
 212        struct clk              *ashb_clk;
 213        int                     irq;
 214        u32                     total_chns;
 215        struct sprd_dma_chn     channels[];
 216};
 217
 218static void sprd_dma_free_desc(struct virt_dma_desc *vd);
 219static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param);
 220static struct of_dma_filter_info sprd_dma_info = {
 221        .filter_fn = sprd_dma_filter_fn,
 222};
 223
 224static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c)
 225{
 226        return container_of(c, struct sprd_dma_chn, vc.chan);
 227}
 228
 229static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c)
 230{
 231        struct sprd_dma_chn *schan = to_sprd_dma_chan(c);
 232
 233        return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);
 234}
 235
 236static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd)
 237{
 238        return container_of(vd, struct sprd_dma_desc, vd);
 239}
 240
 241static void sprd_dma_glb_update(struct sprd_dma_dev *sdev, u32 reg,
 242                                u32 mask, u32 val)
 243{
 244        u32 orig = readl(sdev->glb_base + reg);
 245        u32 tmp;
 246
 247        tmp = (orig & ~mask) | val;
 248        writel(tmp, sdev->glb_base + reg);
 249}
 250
 251static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,
 252                                u32 mask, u32 val)
 253{
 254        u32 orig = readl(schan->chn_base + reg);
 255        u32 tmp;
 256
 257        tmp = (orig & ~mask) | val;
 258        writel(tmp, schan->chn_base + reg);
 259}
 260
 261static int sprd_dma_enable(struct sprd_dma_dev *sdev)
 262{
 263        int ret;
 264
 265        ret = clk_prepare_enable(sdev->clk);
 266        if (ret)
 267                return ret;
 268
 269        /*
 270         * The ashb_clk is optional and only for AGCP DMA controller, so we
 271         * need add one condition to check if the ashb_clk need enable.
 272         */
 273        if (!IS_ERR(sdev->ashb_clk))
 274                ret = clk_prepare_enable(sdev->ashb_clk);
 275
 276        return ret;
 277}
 278
 279static void sprd_dma_disable(struct sprd_dma_dev *sdev)
 280{
 281        clk_disable_unprepare(sdev->clk);
 282
 283        /*
 284         * Need to check if we need disable the optional ashb_clk for AGCP DMA.
 285         */
 286        if (!IS_ERR(sdev->ashb_clk))
 287                clk_disable_unprepare(sdev->ashb_clk);
 288}
 289
 290static void sprd_dma_set_uid(struct sprd_dma_chn *schan)
 291{
 292        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 293        u32 dev_id = schan->dev_id;
 294
 295        if (dev_id != SPRD_DMA_SOFTWARE_UID) {
 296                u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
 297                                 SPRD_DMA_GLB_REQ_UID(dev_id);
 298
 299                writel(schan->chn_num + 1, sdev->glb_base + uid_offset);
 300        }
 301}
 302
 303static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)
 304{
 305        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 306        u32 dev_id = schan->dev_id;
 307
 308        if (dev_id != SPRD_DMA_SOFTWARE_UID) {
 309                u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
 310                                 SPRD_DMA_GLB_REQ_UID(dev_id);
 311
 312                writel(0, sdev->glb_base + uid_offset);
 313        }
 314}
 315
 316static void sprd_dma_clear_int(struct sprd_dma_chn *schan)
 317{
 318        sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,
 319                            SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET,
 320                            SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET);
 321}
 322
 323static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)
 324{
 325        sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,
 326                            SPRD_DMA_CHN_EN);
 327}
 328
 329static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)
 330{
 331        sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);
 332}
 333
 334static void sprd_dma_soft_request(struct sprd_dma_chn *schan)
 335{
 336        sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,
 337                            SPRD_DMA_REQ_EN);
 338}
 339
 340static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)
 341{
 342        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 343        u32 pause, timeout = SPRD_DMA_PAUSE_CNT;
 344
 345        if (enable) {
 346                sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
 347                                    SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN);
 348
 349                do {
 350                        pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);
 351                        if (pause & SPRD_DMA_PAUSE_STS)
 352                                break;
 353
 354                        cpu_relax();
 355                } while (--timeout > 0);
 356
 357                if (!timeout)
 358                        dev_warn(sdev->dma_dev.dev,
 359                                 "pause dma controller timeout\n");
 360        } else {
 361                sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
 362                                    SPRD_DMA_PAUSE_EN, 0);
 363        }
 364}
 365
 366static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)
 367{
 368        u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);
 369
 370        if (!(cfg & SPRD_DMA_CHN_EN))
 371                return;
 372
 373        sprd_dma_pause_resume(schan, true);
 374        sprd_dma_disable_chn(schan);
 375}
 376
 377static unsigned long sprd_dma_get_src_addr(struct sprd_dma_chn *schan)
 378{
 379        unsigned long addr, addr_high;
 380
 381        addr = readl(schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
 382        addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_PTR) &
 383                    SPRD_DMA_HIGH_ADDR_MASK;
 384
 385        return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
 386}
 387
 388static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)
 389{
 390        unsigned long addr, addr_high;
 391
 392        addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
 393        addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &
 394                    SPRD_DMA_HIGH_ADDR_MASK;
 395
 396        return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
 397}
 398
 399static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)
 400{
 401        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 402        u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &
 403                       SPRD_DMA_CHN_INT_STS;
 404
 405        switch (intc_sts) {
 406        case SPRD_DMA_CFGERR_INT_STS:
 407                return SPRD_DMA_CFGERR_INT;
 408
 409        case SPRD_DMA_LIST_INT_STS:
 410                return SPRD_DMA_LIST_INT;
 411
 412        case SPRD_DMA_TRSC_INT_STS:
 413                return SPRD_DMA_TRANS_INT;
 414
 415        case SPRD_DMA_BLK_INT_STS:
 416                return SPRD_DMA_BLK_INT;
 417
 418        case SPRD_DMA_FRAG_INT_STS:
 419                return SPRD_DMA_FRAG_INT;
 420
 421        default:
 422                dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n");
 423                return SPRD_DMA_NO_INT;
 424        }
 425}
 426
 427static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)
 428{
 429        u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
 430
 431        return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK;
 432}
 433
 434static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan)
 435{
 436        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 437        u32 val, chn = schan->chn_num + 1;
 438
 439        switch (schan->chn_mode) {
 440        case SPRD_DMA_SRC_CHN0:
 441                val = chn & SPRD_DMA_GLB_SRC_CHN_MASK;
 442                val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;
 443                val |= SPRD_DMA_GLB_2STAGE_EN;
 444                if (schan->int_type != SPRD_DMA_NO_INT)
 445                        val |= SPRD_DMA_GLB_SRC_INT;
 446
 447                sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val);
 448                break;
 449
 450        case SPRD_DMA_SRC_CHN1:
 451                val = chn & SPRD_DMA_GLB_SRC_CHN_MASK;
 452                val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;
 453                val |= SPRD_DMA_GLB_2STAGE_EN;
 454                if (schan->int_type != SPRD_DMA_NO_INT)
 455                        val |= SPRD_DMA_GLB_SRC_INT;
 456
 457                sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val);
 458                break;
 459
 460        case SPRD_DMA_DST_CHN0:
 461                val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) &
 462                        SPRD_DMA_GLB_DEST_CHN_MASK;
 463                val |= SPRD_DMA_GLB_2STAGE_EN;
 464                if (schan->int_type != SPRD_DMA_NO_INT)
 465                        val |= SPRD_DMA_GLB_DEST_INT;
 466
 467                sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val);
 468                break;
 469
 470        case SPRD_DMA_DST_CHN1:
 471                val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) &
 472                        SPRD_DMA_GLB_DEST_CHN_MASK;
 473                val |= SPRD_DMA_GLB_2STAGE_EN;
 474                if (schan->int_type != SPRD_DMA_NO_INT)
 475                        val |= SPRD_DMA_GLB_DEST_INT;
 476
 477                sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val);
 478                break;
 479
 480        default:
 481                dev_err(sdev->dma_dev.dev, "invalid channel mode setting %d\n",
 482                        schan->chn_mode);
 483                return -EINVAL;
 484        }
 485
 486        return 0;
 487}
 488
 489static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable)
 490{
 491        struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 492        u32 reg, val, req_id;
 493
 494        if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)
 495                return;
 496
 497        /* The DMA request id always starts from 0. */
 498        req_id = schan->dev_id - 1;
 499
 500        if (req_id < 32) {
 501                reg = SPRD_DMA_GLB_REQ_PEND0_EN;
 502                val = BIT(req_id);
 503        } else {
 504                reg = SPRD_DMA_GLB_REQ_PEND1_EN;
 505                val = BIT(req_id - 32);
 506        }
 507
 508        sprd_dma_glb_update(sdev, reg, val, enable ? val : 0);
 509}
 510
 511static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,
 512                                    struct sprd_dma_desc *sdesc)
 513{
 514        struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw;
 515
 516        writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);
 517        writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);
 518        writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);
 519        writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
 520        writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
 521        writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
 522        writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);
 523        writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);
 524        writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);
 525        writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);
 526        writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);
 527        writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);
 528        writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);
 529        writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);
 530        writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);
 531        writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);
 532}
 533
 534static void sprd_dma_start(struct sprd_dma_chn *schan)
 535{
 536        struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);
 537
 538        if (!vd)
 539                return;
 540
 541        list_del(&vd->node);
 542        schan->cur_desc = to_sprd_dma_desc(vd);
 543
 544        /*
 545         * Set 2-stage configuration if the channel starts one 2-stage
 546         * transfer.
 547         */
 548        if (schan->chn_mode && sprd_dma_set_2stage_config(schan))
 549                return;
 550
 551        /*
 552         * Copy the DMA configuration from DMA descriptor to this hardware
 553         * channel.
 554         */
 555        sprd_dma_set_chn_config(schan, schan->cur_desc);
 556        sprd_dma_set_uid(schan);
 557        sprd_dma_set_pending(schan, true);
 558        sprd_dma_enable_chn(schan);
 559
 560        if (schan->dev_id == SPRD_DMA_SOFTWARE_UID &&
 561            schan->chn_mode != SPRD_DMA_DST_CHN0 &&
 562            schan->chn_mode != SPRD_DMA_DST_CHN1)
 563                sprd_dma_soft_request(schan);
 564}
 565
 566static void sprd_dma_stop(struct sprd_dma_chn *schan)
 567{
 568        sprd_dma_stop_and_disable(schan);
 569        sprd_dma_set_pending(schan, false);
 570        sprd_dma_unset_uid(schan);
 571        sprd_dma_clear_int(schan);
 572        schan->cur_desc = NULL;
 573}
 574
 575static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc,
 576                                      enum sprd_dma_int_type int_type,
 577                                      enum sprd_dma_req_mode req_mode)
 578{
 579        if (int_type == SPRD_DMA_NO_INT)
 580                return false;
 581
 582        if (int_type >= req_mode + 1)
 583                return true;
 584        else
 585                return false;
 586}
 587
 588static irqreturn_t dma_irq_handle(int irq, void *dev_id)
 589{
 590        struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id;
 591        u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS);
 592        struct sprd_dma_chn *schan;
 593        struct sprd_dma_desc *sdesc;
 594        enum sprd_dma_req_mode req_type;
 595        enum sprd_dma_int_type int_type;
 596        bool trans_done = false, cyclic = false;
 597        u32 i;
 598
 599        while (irq_status) {
 600                i = __ffs(irq_status);
 601                irq_status &= (irq_status - 1);
 602                schan = &sdev->channels[i];
 603
 604                spin_lock(&schan->vc.lock);
 605
 606                sdesc = schan->cur_desc;
 607                if (!sdesc) {
 608                        spin_unlock(&schan->vc.lock);
 609                        return IRQ_HANDLED;
 610                }
 611
 612                int_type = sprd_dma_get_int_type(schan);
 613                req_type = sprd_dma_get_req_type(schan);
 614                sprd_dma_clear_int(schan);
 615
 616                /* cyclic mode schedule callback */
 617                cyclic = schan->linklist.phy_addr ? true : false;
 618                if (cyclic == true) {
 619                        vchan_cyclic_callback(&sdesc->vd);
 620                } else {
 621                        /* Check if the dma request descriptor is done. */
 622                        trans_done = sprd_dma_check_trans_done(sdesc, int_type,
 623                                                               req_type);
 624                        if (trans_done == true) {
 625                                vchan_cookie_complete(&sdesc->vd);
 626                                schan->cur_desc = NULL;
 627                                sprd_dma_start(schan);
 628                        }
 629                }
 630                spin_unlock(&schan->vc.lock);
 631        }
 632
 633        return IRQ_HANDLED;
 634}
 635
 636static int sprd_dma_alloc_chan_resources(struct dma_chan *chan)
 637{
 638        return pm_runtime_get_sync(chan->device->dev);
 639}
 640
 641static void sprd_dma_free_chan_resources(struct dma_chan *chan)
 642{
 643        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 644        struct virt_dma_desc *cur_vd = NULL;
 645        unsigned long flags;
 646
 647        spin_lock_irqsave(&schan->vc.lock, flags);
 648        if (schan->cur_desc)
 649                cur_vd = &schan->cur_desc->vd;
 650
 651        sprd_dma_stop(schan);
 652        spin_unlock_irqrestore(&schan->vc.lock, flags);
 653
 654        if (cur_vd)
 655                sprd_dma_free_desc(cur_vd);
 656
 657        vchan_free_chan_resources(&schan->vc);
 658        pm_runtime_put(chan->device->dev);
 659}
 660
 661static enum dma_status sprd_dma_tx_status(struct dma_chan *chan,
 662                                          dma_cookie_t cookie,
 663                                          struct dma_tx_state *txstate)
 664{
 665        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 666        struct virt_dma_desc *vd;
 667        unsigned long flags;
 668        enum dma_status ret;
 669        u32 pos;
 670
 671        ret = dma_cookie_status(chan, cookie, txstate);
 672        if (ret == DMA_COMPLETE || !txstate)
 673                return ret;
 674
 675        spin_lock_irqsave(&schan->vc.lock, flags);
 676        vd = vchan_find_desc(&schan->vc, cookie);
 677        if (vd) {
 678                struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
 679                struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
 680
 681                if (hw->trsc_len > 0)
 682                        pos = hw->trsc_len;
 683                else if (hw->blk_len > 0)
 684                        pos = hw->blk_len;
 685                else if (hw->frg_len > 0)
 686                        pos = hw->frg_len;
 687                else
 688                        pos = 0;
 689        } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {
 690                struct sprd_dma_desc *sdesc = schan->cur_desc;
 691
 692                if (sdesc->dir == DMA_DEV_TO_MEM)
 693                        pos = sprd_dma_get_dst_addr(schan);
 694                else
 695                        pos = sprd_dma_get_src_addr(schan);
 696        } else {
 697                pos = 0;
 698        }
 699        spin_unlock_irqrestore(&schan->vc.lock, flags);
 700
 701        dma_set_residue(txstate, pos);
 702        return ret;
 703}
 704
 705static void sprd_dma_issue_pending(struct dma_chan *chan)
 706{
 707        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 708        unsigned long flags;
 709
 710        spin_lock_irqsave(&schan->vc.lock, flags);
 711        if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)
 712                sprd_dma_start(schan);
 713        spin_unlock_irqrestore(&schan->vc.lock, flags);
 714}
 715
 716static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth)
 717{
 718        switch (buswidth) {
 719        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 720        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 721        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 722        case DMA_SLAVE_BUSWIDTH_8_BYTES:
 723                return ffs(buswidth) - 1;
 724
 725        default:
 726                return -EINVAL;
 727        }
 728}
 729
 730static int sprd_dma_get_step(enum dma_slave_buswidth buswidth)
 731{
 732        switch (buswidth) {
 733        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 734        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 735        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 736        case DMA_SLAVE_BUSWIDTH_8_BYTES:
 737                return buswidth;
 738
 739        default:
 740                return -EINVAL;
 741        }
 742}
 743
 744static int sprd_dma_fill_desc(struct dma_chan *chan,
 745                              struct sprd_dma_chn_hw *hw,
 746                              unsigned int sglen, int sg_index,
 747                              dma_addr_t src, dma_addr_t dst, u32 len,
 748                              enum dma_transfer_direction dir,
 749                              unsigned long flags,
 750                              struct dma_slave_config *slave_cfg)
 751{
 752        struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
 753        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 754        enum sprd_dma_chn_mode chn_mode = schan->chn_mode;
 755        u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK;
 756        u32 int_mode = flags & SPRD_DMA_INT_MASK;
 757        int src_datawidth, dst_datawidth, src_step, dst_step;
 758        u32 temp, fix_mode = 0, fix_en = 0;
 759        phys_addr_t llist_ptr;
 760
 761        if (dir == DMA_MEM_TO_DEV) {
 762                src_step = sprd_dma_get_step(slave_cfg->src_addr_width);
 763                if (src_step < 0) {
 764                        dev_err(sdev->dma_dev.dev, "invalid source step\n");
 765                        return src_step;
 766                }
 767
 768                /*
 769                 * For 2-stage transfer, destination channel step can not be 0,
 770                 * since destination device is AON IRAM.
 771                 */
 772                if (chn_mode == SPRD_DMA_DST_CHN0 ||
 773                    chn_mode == SPRD_DMA_DST_CHN1)
 774                        dst_step = src_step;
 775                else
 776                        dst_step = SPRD_DMA_NONE_STEP;
 777        } else {
 778                dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width);
 779                if (dst_step < 0) {
 780                        dev_err(sdev->dma_dev.dev, "invalid destination step\n");
 781                        return dst_step;
 782                }
 783                src_step = SPRD_DMA_NONE_STEP;
 784        }
 785
 786        src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width);
 787        if (src_datawidth < 0) {
 788                dev_err(sdev->dma_dev.dev, "invalid source datawidth\n");
 789                return src_datawidth;
 790        }
 791
 792        dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width);
 793        if (dst_datawidth < 0) {
 794                dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n");
 795                return dst_datawidth;
 796        }
 797
 798        if (slave_cfg->slave_id)
 799                schan->dev_id = slave_cfg->slave_id;
 800
 801        hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
 802
 803        /*
 804         * wrap_ptr and wrap_to will save the high 4 bits source address and
 805         * destination address.
 806         */
 807        hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
 808        hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
 809        hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
 810        hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK;
 811
 812        /*
 813         * If the src step and dst step both are 0 or both are not 0, that means
 814         * we can not enable the fix mode. If one is 0 and another one is not,
 815         * we can enable the fix mode.
 816         */
 817        if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) {
 818                fix_en = 0;
 819        } else {
 820                fix_en = 1;
 821                if (src_step)
 822                        fix_mode = 1;
 823                else
 824                        fix_mode = 0;
 825        }
 826
 827        hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN;
 828
 829        temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
 830        temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
 831        temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET;
 832        temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET;
 833        temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET;
 834        temp |= schan->linklist.wrap_addr ?
 835                SPRD_DMA_WRAP_EN | SPRD_DMA_WRAP_SEL_DEST : 0;
 836        temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK;
 837        hw->frg_len = temp;
 838
 839        hw->blk_len = slave_cfg->src_maxburst & SPRD_DMA_BLK_LEN_MASK;
 840        hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 841
 842        temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
 843        temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
 844        hw->trsf_step = temp;
 845
 846        /* link-list configuration */
 847        if (schan->linklist.phy_addr) {
 848                hw->cfg |= SPRD_DMA_LINKLIST_EN;
 849
 850                /* link-list index */
 851                temp = sglen ? (sg_index + 1) % sglen : 0;
 852
 853                /* Next link-list configuration's physical address offset */
 854                temp = temp * sizeof(*hw) + SPRD_DMA_CHN_SRC_ADDR;
 855                /*
 856                 * Set the link-list pointer point to next link-list
 857                 * configuration's physical address.
 858                 */
 859                llist_ptr = schan->linklist.phy_addr + temp;
 860                hw->llist_ptr = lower_32_bits(llist_ptr);
 861                hw->src_blk_step = (upper_32_bits(llist_ptr) << SPRD_DMA_LLIST_HIGH_SHIFT) &
 862                        SPRD_DMA_LLIST_HIGH_MASK;
 863
 864                if (schan->linklist.wrap_addr) {
 865                        hw->wrap_ptr |= schan->linklist.wrap_addr &
 866                                SPRD_DMA_WRAP_ADDR_MASK;
 867                        hw->wrap_to |= dst & SPRD_DMA_WRAP_ADDR_MASK;
 868                }
 869        } else {
 870                hw->llist_ptr = 0;
 871                hw->src_blk_step = 0;
 872        }
 873
 874        hw->frg_step = 0;
 875        hw->des_blk_step = 0;
 876        return 0;
 877}
 878
 879static int sprd_dma_fill_linklist_desc(struct dma_chan *chan,
 880                                       unsigned int sglen, int sg_index,
 881                                       dma_addr_t src, dma_addr_t dst, u32 len,
 882                                       enum dma_transfer_direction dir,
 883                                       unsigned long flags,
 884                                       struct dma_slave_config *slave_cfg)
 885{
 886        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 887        struct sprd_dma_chn_hw *hw;
 888
 889        if (!schan->linklist.virt_addr)
 890                return -EINVAL;
 891
 892        hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr +
 893                                        sg_index * sizeof(*hw));
 894
 895        return sprd_dma_fill_desc(chan, hw, sglen, sg_index, src, dst, len,
 896                                  dir, flags, slave_cfg);
 897}
 898
 899static struct dma_async_tx_descriptor *
 900sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 901                         size_t len, unsigned long flags)
 902{
 903        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 904        struct sprd_dma_desc *sdesc;
 905        struct sprd_dma_chn_hw *hw;
 906        enum sprd_dma_datawidth datawidth;
 907        u32 step, temp;
 908
 909        sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
 910        if (!sdesc)
 911                return NULL;
 912
 913        hw = &sdesc->chn_hw;
 914
 915        hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
 916        hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN;
 917        hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
 918        hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK;
 919        hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
 920                SPRD_DMA_HIGH_ADDR_MASK;
 921        hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
 922                SPRD_DMA_HIGH_ADDR_MASK;
 923
 924        if (IS_ALIGNED(len, 8)) {
 925                datawidth = SPRD_DMA_DATAWIDTH_8_BYTES;
 926                step = SPRD_DMA_DWORD_STEP;
 927        } else if (IS_ALIGNED(len, 4)) {
 928                datawidth = SPRD_DMA_DATAWIDTH_4_BYTES;
 929                step = SPRD_DMA_WORD_STEP;
 930        } else if (IS_ALIGNED(len, 2)) {
 931                datawidth = SPRD_DMA_DATAWIDTH_2_BYTES;
 932                step = SPRD_DMA_SHORT_STEP;
 933        } else {
 934                datawidth = SPRD_DMA_DATAWIDTH_1_BYTE;
 935                step = SPRD_DMA_BYTE_STEP;
 936        }
 937
 938        temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
 939        temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
 940        temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET;
 941        temp |= len & SPRD_DMA_FRG_LEN_MASK;
 942        hw->frg_len = temp;
 943
 944        hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
 945        hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 946
 947        temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
 948        temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
 949        hw->trsf_step = temp;
 950
 951        return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
 952}
 953
 954static struct dma_async_tx_descriptor *
 955sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 956                       unsigned int sglen, enum dma_transfer_direction dir,
 957                       unsigned long flags, void *context)
 958{
 959        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 960        struct dma_slave_config *slave_cfg = &schan->slave_cfg;
 961        dma_addr_t src = 0, dst = 0;
 962        dma_addr_t start_src = 0, start_dst = 0;
 963        struct sprd_dma_desc *sdesc;
 964        struct scatterlist *sg;
 965        u32 len = 0;
 966        int ret, i;
 967
 968        if (!is_slave_direction(dir))
 969                return NULL;
 970
 971        if (context) {
 972                struct sprd_dma_linklist *ll_cfg =
 973                        (struct sprd_dma_linklist *)context;
 974
 975                schan->linklist.phy_addr = ll_cfg->phy_addr;
 976                schan->linklist.virt_addr = ll_cfg->virt_addr;
 977                schan->linklist.wrap_addr = ll_cfg->wrap_addr;
 978        } else {
 979                schan->linklist.phy_addr = 0;
 980                schan->linklist.virt_addr = 0;
 981                schan->linklist.wrap_addr = 0;
 982        }
 983
 984        /*
 985         * Set channel mode, interrupt mode and trigger mode for 2-stage
 986         * transfer.
 987         */
 988        schan->chn_mode =
 989                (flags >> SPRD_DMA_CHN_MODE_SHIFT) & SPRD_DMA_CHN_MODE_MASK;
 990        schan->trg_mode =
 991                (flags >> SPRD_DMA_TRG_MODE_SHIFT) & SPRD_DMA_TRG_MODE_MASK;
 992        schan->int_type = flags & SPRD_DMA_INT_TYPE_MASK;
 993
 994        sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
 995        if (!sdesc)
 996                return NULL;
 997
 998        sdesc->dir = dir;
 999
1000        for_each_sg(sgl, sg, sglen, i) {
1001                len = sg_dma_len(sg);
1002
1003                if (dir == DMA_MEM_TO_DEV) {
1004                        src = sg_dma_address(sg);
1005                        dst = slave_cfg->dst_addr;
1006                } else {
1007                        src = slave_cfg->src_addr;
1008                        dst = sg_dma_address(sg);
1009                }
1010
1011                if (!i) {
1012                        start_src = src;
1013                        start_dst = dst;
1014                }
1015
1016                /*
1017                 * The link-list mode needs at least 2 link-list
1018                 * configurations. If there is only one sg, it doesn't
1019                 * need to fill the link-list configuration.
1020                 */
1021                if (sglen < 2)
1022                        break;
1023
1024                ret = sprd_dma_fill_linklist_desc(chan, sglen, i, src, dst, len,
1025                                                  dir, flags, slave_cfg);
1026                if (ret) {
1027                        kfree(sdesc);
1028                        return NULL;
1029                }
1030        }
1031
1032        ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, start_src,
1033                                 start_dst, len, dir, flags, slave_cfg);
1034        if (ret) {
1035                kfree(sdesc);
1036                return NULL;
1037        }
1038
1039        return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
1040}
1041
1042static int sprd_dma_slave_config(struct dma_chan *chan,
1043                                 struct dma_slave_config *config)
1044{
1045        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1046        struct dma_slave_config *slave_cfg = &schan->slave_cfg;
1047
1048        memcpy(slave_cfg, config, sizeof(*config));
1049        return 0;
1050}
1051
1052static int sprd_dma_pause(struct dma_chan *chan)
1053{
1054        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1055        unsigned long flags;
1056
1057        spin_lock_irqsave(&schan->vc.lock, flags);
1058        sprd_dma_pause_resume(schan, true);
1059        spin_unlock_irqrestore(&schan->vc.lock, flags);
1060
1061        return 0;
1062}
1063
1064static int sprd_dma_resume(struct dma_chan *chan)
1065{
1066        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1067        unsigned long flags;
1068
1069        spin_lock_irqsave(&schan->vc.lock, flags);
1070        sprd_dma_pause_resume(schan, false);
1071        spin_unlock_irqrestore(&schan->vc.lock, flags);
1072
1073        return 0;
1074}
1075
1076static int sprd_dma_terminate_all(struct dma_chan *chan)
1077{
1078        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1079        struct virt_dma_desc *cur_vd = NULL;
1080        unsigned long flags;
1081        LIST_HEAD(head);
1082
1083        spin_lock_irqsave(&schan->vc.lock, flags);
1084        if (schan->cur_desc)
1085                cur_vd = &schan->cur_desc->vd;
1086
1087        sprd_dma_stop(schan);
1088
1089        vchan_get_all_descriptors(&schan->vc, &head);
1090        spin_unlock_irqrestore(&schan->vc.lock, flags);
1091
1092        if (cur_vd)
1093                sprd_dma_free_desc(cur_vd);
1094
1095        vchan_dma_desc_free_list(&schan->vc, &head);
1096        return 0;
1097}
1098
1099static void sprd_dma_free_desc(struct virt_dma_desc *vd)
1100{
1101        struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
1102
1103        kfree(sdesc);
1104}
1105
1106static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param)
1107{
1108        struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1109        u32 slave_id = *(u32 *)param;
1110
1111        schan->dev_id = slave_id;
1112        return true;
1113}
1114
1115static int sprd_dma_probe(struct platform_device *pdev)
1116{
1117        struct device_node *np = pdev->dev.of_node;
1118        struct sprd_dma_dev *sdev;
1119        struct sprd_dma_chn *dma_chn;
1120        u32 chn_count;
1121        int ret, i;
1122
1123        ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count);
1124        if (ret) {
1125                dev_err(&pdev->dev, "get dma channels count failed\n");
1126                return ret;
1127        }
1128
1129        sdev = devm_kzalloc(&pdev->dev,
1130                            struct_size(sdev, channels, chn_count),
1131                            GFP_KERNEL);
1132        if (!sdev)
1133                return -ENOMEM;
1134
1135        sdev->clk = devm_clk_get(&pdev->dev, "enable");
1136        if (IS_ERR(sdev->clk)) {
1137                dev_err(&pdev->dev, "get enable clock failed\n");
1138                return PTR_ERR(sdev->clk);
1139        }
1140
1141        /* ashb clock is optional for AGCP DMA */
1142        sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
1143        if (IS_ERR(sdev->ashb_clk))
1144                dev_warn(&pdev->dev, "no optional ashb eb clock\n");
1145
1146        /*
1147         * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP
1148         * DMA controller, it can or do not request the irq, which will save
1149         * system power without resuming system by DMA interrupts if AGCP DMA
1150         * does not request the irq. Thus the DMA interrupts property should
1151         * be optional.
1152         */
1153        sdev->irq = platform_get_irq(pdev, 0);
1154        if (sdev->irq > 0) {
1155                ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle,
1156                                       0, "sprd_dma", (void *)sdev);
1157                if (ret < 0) {
1158                        dev_err(&pdev->dev, "request dma irq failed\n");
1159                        return ret;
1160                }
1161        } else {
1162                dev_warn(&pdev->dev, "no interrupts for the dma controller\n");
1163        }
1164
1165        sdev->glb_base = devm_platform_ioremap_resource(pdev, 0);
1166        if (IS_ERR(sdev->glb_base))
1167                return PTR_ERR(sdev->glb_base);
1168
1169        dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
1170        sdev->total_chns = chn_count;
1171        sdev->dma_dev.chancnt = chn_count;
1172        INIT_LIST_HEAD(&sdev->dma_dev.channels);
1173        INIT_LIST_HEAD(&sdev->dma_dev.global_node);
1174        sdev->dma_dev.dev = &pdev->dev;
1175        sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources;
1176        sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources;
1177        sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
1178        sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
1179        sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
1180        sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg;
1181        sdev->dma_dev.device_config = sprd_dma_slave_config;
1182        sdev->dma_dev.device_pause = sprd_dma_pause;
1183        sdev->dma_dev.device_resume = sprd_dma_resume;
1184        sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all;
1185
1186        for (i = 0; i < chn_count; i++) {
1187                dma_chn = &sdev->channels[i];
1188                dma_chn->chn_num = i;
1189                dma_chn->cur_desc = NULL;
1190                /* get each channel's registers base address. */
1191                dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET +
1192                                    SPRD_DMA_CHN_REG_LENGTH * i;
1193
1194                dma_chn->vc.desc_free = sprd_dma_free_desc;
1195                vchan_init(&dma_chn->vc, &sdev->dma_dev);
1196        }
1197
1198        platform_set_drvdata(pdev, sdev);
1199        ret = sprd_dma_enable(sdev);
1200        if (ret)
1201                return ret;
1202
1203        pm_runtime_set_active(&pdev->dev);
1204        pm_runtime_enable(&pdev->dev);
1205
1206        ret = pm_runtime_get_sync(&pdev->dev);
1207        if (ret < 0)
1208                goto err_rpm;
1209
1210        ret = dma_async_device_register(&sdev->dma_dev);
1211        if (ret < 0) {
1212                dev_err(&pdev->dev, "register dma device failed:%d\n", ret);
1213                goto err_register;
1214        }
1215
1216        sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask;
1217        ret = of_dma_controller_register(np, of_dma_simple_xlate,
1218                                         &sprd_dma_info);
1219        if (ret)
1220                goto err_of_register;
1221
1222        pm_runtime_put(&pdev->dev);
1223        return 0;
1224
1225err_of_register:
1226        dma_async_device_unregister(&sdev->dma_dev);
1227err_register:
1228        pm_runtime_put_noidle(&pdev->dev);
1229        pm_runtime_disable(&pdev->dev);
1230err_rpm:
1231        sprd_dma_disable(sdev);
1232        return ret;
1233}
1234
1235static int sprd_dma_remove(struct platform_device *pdev)
1236{
1237        struct sprd_dma_dev *sdev = platform_get_drvdata(pdev);
1238        struct sprd_dma_chn *c, *cn;
1239        int ret;
1240
1241        ret = pm_runtime_get_sync(&pdev->dev);
1242        if (ret < 0)
1243                return ret;
1244
1245        /* explicitly free the irq */
1246        if (sdev->irq > 0)
1247                devm_free_irq(&pdev->dev, sdev->irq, sdev);
1248
1249        list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels,
1250                                 vc.chan.device_node) {
1251                list_del(&c->vc.chan.device_node);
1252                tasklet_kill(&c->vc.task);
1253        }
1254
1255        of_dma_controller_free(pdev->dev.of_node);
1256        dma_async_device_unregister(&sdev->dma_dev);
1257        sprd_dma_disable(sdev);
1258
1259        pm_runtime_put_noidle(&pdev->dev);
1260        pm_runtime_disable(&pdev->dev);
1261        return 0;
1262}
1263
1264static const struct of_device_id sprd_dma_match[] = {
1265        { .compatible = "sprd,sc9860-dma", },
1266        {},
1267};
1268
1269static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
1270{
1271        struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1272
1273        sprd_dma_disable(sdev);
1274        return 0;
1275}
1276
1277static int __maybe_unused sprd_dma_runtime_resume(struct device *dev)
1278{
1279        struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1280        int ret;
1281
1282        ret = sprd_dma_enable(sdev);
1283        if (ret)
1284                dev_err(sdev->dma_dev.dev, "enable dma failed\n");
1285
1286        return ret;
1287}
1288
1289static const struct dev_pm_ops sprd_dma_pm_ops = {
1290        SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend,
1291                           sprd_dma_runtime_resume,
1292                           NULL)
1293};
1294
1295static struct platform_driver sprd_dma_driver = {
1296        .probe = sprd_dma_probe,
1297        .remove = sprd_dma_remove,
1298        .driver = {
1299                .name = "sprd-dma",
1300                .of_match_table = sprd_dma_match,
1301                .pm = &sprd_dma_pm_ops,
1302        },
1303};
1304module_platform_driver(sprd_dma_driver);
1305
1306MODULE_LICENSE("GPL v2");
1307MODULE_DESCRIPTION("DMA driver for Spreadtrum");
1308MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
1309MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>");
1310MODULE_ALIAS("platform:sprd-dma");
1311