linux/drivers/staging/mt7621-dma/mtk-hsdma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 *  Copyright (C) 2015, Michael Lee <igvtee@gmail.com>
   4 *  MTK HSDMA support
   5 */
   6
   7#include <linux/dmaengine.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/err.h>
  10#include <linux/init.h>
  11#include <linux/list.h>
  12#include <linux/module.h>
  13#include <linux/platform_device.h>
  14#include <linux/slab.h>
  15#include <linux/spinlock.h>
  16#include <linux/irq.h>
  17#include <linux/of_dma.h>
  18#include <linux/reset.h>
  19#include <linux/of_device.h>
  20
  21#include "virt-dma.h"
  22
  23#define HSDMA_BASE_OFFSET               0x800
  24
  25#define HSDMA_REG_TX_BASE               0x00
  26#define HSDMA_REG_TX_CNT                0x04
  27#define HSDMA_REG_TX_CTX                0x08
  28#define HSDMA_REG_TX_DTX                0x0c
  29#define HSDMA_REG_RX_BASE               0x100
  30#define HSDMA_REG_RX_CNT                0x104
  31#define HSDMA_REG_RX_CRX                0x108
  32#define HSDMA_REG_RX_DRX                0x10c
  33#define HSDMA_REG_INFO                  0x200
  34#define HSDMA_REG_GLO_CFG               0x204
  35#define HSDMA_REG_RST_CFG               0x208
  36#define HSDMA_REG_DELAY_INT             0x20c
  37#define HSDMA_REG_FREEQ_THRES           0x210
  38#define HSDMA_REG_INT_STATUS            0x220
  39#define HSDMA_REG_INT_MASK              0x228
  40#define HSDMA_REG_SCH_Q01               0x280
  41#define HSDMA_REG_SCH_Q23               0x284
  42
  43#define HSDMA_DESCS_MAX                 0xfff
  44#define HSDMA_DESCS_NUM                 8
  45#define HSDMA_DESCS_MASK                (HSDMA_DESCS_NUM - 1)
  46#define HSDMA_NEXT_DESC(x)              (((x) + 1) & HSDMA_DESCS_MASK)
  47
  48/* HSDMA_REG_INFO */
  49#define HSDMA_INFO_INDEX_MASK           0xf
  50#define HSDMA_INFO_INDEX_SHIFT          24
  51#define HSDMA_INFO_BASE_MASK            0xff
  52#define HSDMA_INFO_BASE_SHIFT           16
  53#define HSDMA_INFO_RX_MASK              0xff
  54#define HSDMA_INFO_RX_SHIFT             8
  55#define HSDMA_INFO_TX_MASK              0xff
  56#define HSDMA_INFO_TX_SHIFT             0
  57
  58/* HSDMA_REG_GLO_CFG */
  59#define HSDMA_GLO_TX_2B_OFFSET          BIT(31)
  60#define HSDMA_GLO_CLK_GATE              BIT(30)
  61#define HSDMA_GLO_BYTE_SWAP             BIT(29)
  62#define HSDMA_GLO_MULTI_DMA             BIT(10)
  63#define HSDMA_GLO_TWO_BUF               BIT(9)
  64#define HSDMA_GLO_32B_DESC              BIT(8)
  65#define HSDMA_GLO_BIG_ENDIAN            BIT(7)
  66#define HSDMA_GLO_TX_DONE               BIT(6)
  67#define HSDMA_GLO_BT_MASK               0x3
  68#define HSDMA_GLO_BT_SHIFT              4
  69#define HSDMA_GLO_RX_BUSY               BIT(3)
  70#define HSDMA_GLO_RX_DMA                BIT(2)
  71#define HSDMA_GLO_TX_BUSY               BIT(1)
  72#define HSDMA_GLO_TX_DMA                BIT(0)
  73
  74#define HSDMA_BT_SIZE_16BYTES           (0 << HSDMA_GLO_BT_SHIFT)
  75#define HSDMA_BT_SIZE_32BYTES           (1 << HSDMA_GLO_BT_SHIFT)
  76#define HSDMA_BT_SIZE_64BYTES           (2 << HSDMA_GLO_BT_SHIFT)
  77#define HSDMA_BT_SIZE_128BYTES          (3 << HSDMA_GLO_BT_SHIFT)
  78
  79#define HSDMA_GLO_DEFAULT               (HSDMA_GLO_MULTI_DMA | \
  80                HSDMA_GLO_RX_DMA | HSDMA_GLO_TX_DMA | HSDMA_BT_SIZE_32BYTES)
  81
  82/* HSDMA_REG_RST_CFG */
  83#define HSDMA_RST_RX_SHIFT              16
  84#define HSDMA_RST_TX_SHIFT              0
  85
  86/* HSDMA_REG_DELAY_INT */
  87#define HSDMA_DELAY_INT_EN              BIT(15)
  88#define HSDMA_DELAY_PEND_OFFSET         8
  89#define HSDMA_DELAY_TIME_OFFSET         0
  90#define HSDMA_DELAY_TX_OFFSET           16
  91#define HSDMA_DELAY_RX_OFFSET           0
  92
  93#define HSDMA_DELAY_INIT(x)             (HSDMA_DELAY_INT_EN | \
  94                ((x) << HSDMA_DELAY_PEND_OFFSET))
  95#define HSDMA_DELAY(x)                  ((HSDMA_DELAY_INIT(x) << \
  96                HSDMA_DELAY_TX_OFFSET) | HSDMA_DELAY_INIT(x))
  97
  98/* HSDMA_REG_INT_STATUS */
  99#define HSDMA_INT_DELAY_RX_COH          BIT(31)
 100#define HSDMA_INT_DELAY_RX_INT          BIT(30)
 101#define HSDMA_INT_DELAY_TX_COH          BIT(29)
 102#define HSDMA_INT_DELAY_TX_INT          BIT(28)
 103#define HSDMA_INT_RX_MASK               0x3
 104#define HSDMA_INT_RX_SHIFT              16
 105#define HSDMA_INT_RX_Q0                 BIT(16)
 106#define HSDMA_INT_TX_MASK               0xf
 107#define HSDMA_INT_TX_SHIFT              0
 108#define HSDMA_INT_TX_Q0                 BIT(0)
 109
 110/* tx/rx dma desc flags */
 111#define HSDMA_PLEN_MASK                 0x3fff
 112#define HSDMA_DESC_DONE                 BIT(31)
 113#define HSDMA_DESC_LS0                  BIT(30)
 114#define HSDMA_DESC_PLEN0(_x)            (((_x) & HSDMA_PLEN_MASK) << 16)
 115#define HSDMA_DESC_TAG                  BIT(15)
 116#define HSDMA_DESC_LS1                  BIT(14)
 117#define HSDMA_DESC_PLEN1(_x)            ((_x) & HSDMA_PLEN_MASK)
 118
 119/* align 4 bytes */
 120#define HSDMA_ALIGN_SIZE                3
 121/* align size 128bytes */
 122#define HSDMA_MAX_PLEN                  0x3f80
 123
 124struct hsdma_desc {
 125        u32 addr0;
 126        u32 flags;
 127        u32 addr1;
 128        u32 unused;
 129};
 130
 131struct mtk_hsdma_sg {
 132        dma_addr_t src_addr;
 133        dma_addr_t dst_addr;
 134        u32 len;
 135};
 136
 137struct mtk_hsdma_desc {
 138        struct virt_dma_desc vdesc;
 139        unsigned int num_sgs;
 140        struct mtk_hsdma_sg sg[1];
 141};
 142
 143struct mtk_hsdma_chan {
 144        struct virt_dma_chan vchan;
 145        unsigned int id;
 146        dma_addr_t desc_addr;
 147        int tx_idx;
 148        int rx_idx;
 149        struct hsdma_desc *tx_ring;
 150        struct hsdma_desc *rx_ring;
 151        struct mtk_hsdma_desc *desc;
 152        unsigned int next_sg;
 153};
 154
 155struct mtk_hsdam_engine {
 156        struct dma_device ddev;
 157        struct device_dma_parameters dma_parms;
 158        void __iomem *base;
 159        struct tasklet_struct task;
 160        volatile unsigned long chan_issued;
 161
 162        struct mtk_hsdma_chan chan[1];
 163};
 164
 165static inline struct mtk_hsdam_engine *mtk_hsdma_chan_get_dev(
 166                struct mtk_hsdma_chan *chan)
 167{
 168        return container_of(chan->vchan.chan.device, struct mtk_hsdam_engine,
 169                        ddev);
 170}
 171
 172static inline struct mtk_hsdma_chan *to_mtk_hsdma_chan(struct dma_chan *c)
 173{
 174        return container_of(c, struct mtk_hsdma_chan, vchan.chan);
 175}
 176
 177static inline struct mtk_hsdma_desc *to_mtk_hsdma_desc(
 178                struct virt_dma_desc *vdesc)
 179{
 180        return container_of(vdesc, struct mtk_hsdma_desc, vdesc);
 181}
 182
 183static inline u32 mtk_hsdma_read(struct mtk_hsdam_engine *hsdma, u32 reg)
 184{
 185        return readl(hsdma->base + reg);
 186}
 187
 188static inline void mtk_hsdma_write(struct mtk_hsdam_engine *hsdma,
 189                                   unsigned int reg, u32 val)
 190{
 191        writel(val, hsdma->base + reg);
 192}
 193
 194static void mtk_hsdma_reset_chan(struct mtk_hsdam_engine *hsdma,
 195                                 struct mtk_hsdma_chan *chan)
 196{
 197        chan->tx_idx = 0;
 198        chan->rx_idx = HSDMA_DESCS_NUM - 1;
 199
 200        mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx);
 201        mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx);
 202
 203        mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG,
 204                        0x1 << (chan->id + HSDMA_RST_TX_SHIFT));
 205        mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG,
 206                        0x1 << (chan->id + HSDMA_RST_RX_SHIFT));
 207}
 208
 209static void hsdma_dump_reg(struct mtk_hsdam_engine *hsdma)
 210{
 211        dev_dbg(hsdma->ddev.dev, "tbase %08x, tcnt %08x, " \
 212                        "tctx %08x, tdtx: %08x, rbase %08x, " \
 213                        "rcnt %08x, rctx %08x, rdtx %08x\n",
 214                        mtk_hsdma_read(hsdma, HSDMA_REG_TX_BASE),
 215                        mtk_hsdma_read(hsdma, HSDMA_REG_TX_CNT),
 216                        mtk_hsdma_read(hsdma, HSDMA_REG_TX_CTX),
 217                        mtk_hsdma_read(hsdma, HSDMA_REG_TX_DTX),
 218                        mtk_hsdma_read(hsdma, HSDMA_REG_RX_BASE),
 219                        mtk_hsdma_read(hsdma, HSDMA_REG_RX_CNT),
 220                        mtk_hsdma_read(hsdma, HSDMA_REG_RX_CRX),
 221                        mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX));
 222
 223        dev_dbg(hsdma->ddev.dev, "info %08x, glo %08x, delay %08x, " \
 224                        "intr_stat %08x, intr_mask %08x\n",
 225                        mtk_hsdma_read(hsdma, HSDMA_REG_INFO),
 226                        mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG),
 227                        mtk_hsdma_read(hsdma, HSDMA_REG_DELAY_INT),
 228                        mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS),
 229                        mtk_hsdma_read(hsdma, HSDMA_REG_INT_MASK));
 230}
 231
 232static void hsdma_dump_desc(struct mtk_hsdam_engine *hsdma,
 233                            struct mtk_hsdma_chan *chan)
 234{
 235        struct hsdma_desc *tx_desc;
 236        struct hsdma_desc *rx_desc;
 237        int i;
 238
 239        dev_dbg(hsdma->ddev.dev, "tx idx: %d, rx idx: %d\n",
 240                chan->tx_idx, chan->rx_idx);
 241
 242        for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 243                tx_desc = &chan->tx_ring[i];
 244                rx_desc = &chan->rx_ring[i];
 245
 246                dev_dbg(hsdma->ddev.dev, "%d tx addr0: %08x, flags %08x, " \
 247                                "tx addr1: %08x, rx addr0 %08x, flags %08x\n",
 248                                i, tx_desc->addr0, tx_desc->flags, \
 249                                tx_desc->addr1, rx_desc->addr0, rx_desc->flags);
 250        }
 251}
 252
 253static void mtk_hsdma_reset(struct mtk_hsdam_engine *hsdma,
 254                            struct mtk_hsdma_chan *chan)
 255{
 256        int i;
 257
 258        /* disable dma */
 259        mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0);
 260
 261        /* disable intr */
 262        mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0);
 263
 264        /* init desc value */
 265        for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 266                chan->tx_ring[i].addr0 = 0;
 267                chan->tx_ring[i].flags = HSDMA_DESC_LS0 | HSDMA_DESC_DONE;
 268        }
 269        for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 270                chan->rx_ring[i].addr0 = 0;
 271                chan->rx_ring[i].flags = 0;
 272        }
 273
 274        /* reset */
 275        mtk_hsdma_reset_chan(hsdma, chan);
 276
 277        /* enable intr */
 278        mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0);
 279
 280        /* enable dma */
 281        mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT);
 282}
 283
 284static int mtk_hsdma_terminate_all(struct dma_chan *c)
 285{
 286        struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 287        struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan);
 288        unsigned long timeout;
 289        LIST_HEAD(head);
 290
 291        spin_lock_bh(&chan->vchan.lock);
 292        chan->desc = NULL;
 293        clear_bit(chan->id, &hsdma->chan_issued);
 294        vchan_get_all_descriptors(&chan->vchan, &head);
 295        spin_unlock_bh(&chan->vchan.lock);
 296
 297        vchan_dma_desc_free_list(&chan->vchan, &head);
 298
 299        /* wait dma transfer complete */
 300        timeout = jiffies + msecs_to_jiffies(2000);
 301        while (mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG) &
 302                        (HSDMA_GLO_RX_BUSY | HSDMA_GLO_TX_BUSY)) {
 303                if (time_after_eq(jiffies, timeout)) {
 304                        hsdma_dump_desc(hsdma, chan);
 305                        mtk_hsdma_reset(hsdma, chan);
 306                        dev_err(hsdma->ddev.dev, "timeout, reset it\n");
 307                        break;
 308                }
 309                cpu_relax();
 310        }
 311
 312        return 0;
 313}
 314
 315static int mtk_hsdma_start_transfer(struct mtk_hsdam_engine *hsdma,
 316                                    struct mtk_hsdma_chan *chan)
 317{
 318        dma_addr_t src, dst;
 319        size_t len, tlen;
 320        struct hsdma_desc *tx_desc, *rx_desc;
 321        struct mtk_hsdma_sg *sg;
 322        unsigned int i;
 323        int rx_idx;
 324
 325        sg = &chan->desc->sg[0];
 326        len = sg->len;
 327        chan->desc->num_sgs = DIV_ROUND_UP(len, HSDMA_MAX_PLEN);
 328
 329        /* tx desc */
 330        src = sg->src_addr;
 331        for (i = 0; i < chan->desc->num_sgs; i++) {
 332                tx_desc = &chan->tx_ring[chan->tx_idx];
 333
 334                if (len > HSDMA_MAX_PLEN)
 335                        tlen = HSDMA_MAX_PLEN;
 336                else
 337                        tlen = len;
 338
 339                if (i & 0x1) {
 340                        tx_desc->addr1 = src;
 341                        tx_desc->flags |= HSDMA_DESC_PLEN1(tlen);
 342                } else {
 343                        tx_desc->addr0 = src;
 344                        tx_desc->flags = HSDMA_DESC_PLEN0(tlen);
 345
 346                        /* update index */
 347                        chan->tx_idx = HSDMA_NEXT_DESC(chan->tx_idx);
 348                }
 349
 350                src += tlen;
 351                len -= tlen;
 352        }
 353        if (i & 0x1)
 354                tx_desc->flags |= HSDMA_DESC_LS0;
 355        else
 356                tx_desc->flags |= HSDMA_DESC_LS1;
 357
 358        /* rx desc */
 359        rx_idx = HSDMA_NEXT_DESC(chan->rx_idx);
 360        len = sg->len;
 361        dst = sg->dst_addr;
 362        for (i = 0; i < chan->desc->num_sgs; i++) {
 363                rx_desc = &chan->rx_ring[rx_idx];
 364                if (len > HSDMA_MAX_PLEN)
 365                        tlen = HSDMA_MAX_PLEN;
 366                else
 367                        tlen = len;
 368
 369                rx_desc->addr0 = dst;
 370                rx_desc->flags = HSDMA_DESC_PLEN0(tlen);
 371
 372                dst += tlen;
 373                len -= tlen;
 374
 375                /* update index */
 376                rx_idx = HSDMA_NEXT_DESC(rx_idx);
 377        }
 378
 379        /* make sure desc and index all up to date */
 380        wmb();
 381        mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx);
 382
 383        return 0;
 384}
 385
 386static int gdma_next_desc(struct mtk_hsdma_chan *chan)
 387{
 388        struct virt_dma_desc *vdesc;
 389
 390        vdesc = vchan_next_desc(&chan->vchan);
 391        if (!vdesc) {
 392                chan->desc = NULL;
 393                return 0;
 394        }
 395        chan->desc = to_mtk_hsdma_desc(vdesc);
 396        chan->next_sg = 0;
 397
 398        return 1;
 399}
 400
 401static void mtk_hsdma_chan_done(struct mtk_hsdam_engine *hsdma,
 402                                struct mtk_hsdma_chan *chan)
 403{
 404        struct mtk_hsdma_desc *desc;
 405        int chan_issued;
 406
 407        chan_issued = 0;
 408        spin_lock_bh(&chan->vchan.lock);
 409        desc = chan->desc;
 410        if (likely(desc)) {
 411                if (chan->next_sg == desc->num_sgs) {
 412                        list_del(&desc->vdesc.node);
 413                        vchan_cookie_complete(&desc->vdesc);
 414                        chan_issued = gdma_next_desc(chan);
 415                }
 416        } else {
 417                dev_dbg(hsdma->ddev.dev, "no desc to complete\n");
 418        }
 419
 420        if (chan_issued)
 421                set_bit(chan->id, &hsdma->chan_issued);
 422        spin_unlock_bh(&chan->vchan.lock);
 423}
 424
 425static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
 426{
 427        struct mtk_hsdam_engine *hsdma = devid;
 428        u32 status;
 429
 430        status = mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS);
 431        if (unlikely(!status))
 432                return IRQ_NONE;
 433
 434        if (likely(status & HSDMA_INT_RX_Q0))
 435                tasklet_schedule(&hsdma->task);
 436        else
 437                dev_dbg(hsdma->ddev.dev, "unhandle irq status %08x\n", status);
 438        /* clean intr bits */
 439        mtk_hsdma_write(hsdma, HSDMA_REG_INT_STATUS, status);
 440
 441        return IRQ_HANDLED;
 442}
 443
 444static void mtk_hsdma_issue_pending(struct dma_chan *c)
 445{
 446        struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 447        struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan);
 448
 449        spin_lock_bh(&chan->vchan.lock);
 450        if (vchan_issue_pending(&chan->vchan) && !chan->desc) {
 451                if (gdma_next_desc(chan)) {
 452                        set_bit(chan->id, &hsdma->chan_issued);
 453                        tasklet_schedule(&hsdma->task);
 454                } else {
 455                        dev_dbg(hsdma->ddev.dev, "no desc to issue\n");
 456                }
 457        }
 458        spin_unlock_bh(&chan->vchan.lock);
 459}
 460
 461static struct dma_async_tx_descriptor *mtk_hsdma_prep_dma_memcpy(
 462                struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
 463                size_t len, unsigned long flags)
 464{
 465        struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 466        struct mtk_hsdma_desc *desc;
 467
 468        if (len <= 0)
 469                return NULL;
 470
 471        desc = kzalloc(sizeof(*desc), GFP_ATOMIC);
 472        if (!desc) {
 473                dev_err(c->device->dev, "alloc memcpy decs error\n");
 474                return NULL;
 475        }
 476
 477        desc->sg[0].src_addr = src;
 478        desc->sg[0].dst_addr = dest;
 479        desc->sg[0].len = len;
 480
 481        return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 482}
 483
 484static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
 485                                           dma_cookie_t cookie,
 486                                           struct dma_tx_state *state)
 487{
 488        return dma_cookie_status(c, cookie, state);
 489}
 490
 491static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
 492{
 493        vchan_free_chan_resources(to_virt_chan(c));
 494}
 495
 496static void mtk_hsdma_desc_free(struct virt_dma_desc *vdesc)
 497{
 498        kfree(container_of(vdesc, struct mtk_hsdma_desc, vdesc));
 499}
 500
 501static void mtk_hsdma_tx(struct mtk_hsdam_engine *hsdma)
 502{
 503        struct mtk_hsdma_chan *chan;
 504
 505        if (test_and_clear_bit(0, &hsdma->chan_issued)) {
 506                chan = &hsdma->chan[0];
 507                if (chan->desc)
 508                        mtk_hsdma_start_transfer(hsdma, chan);
 509                else
 510                        dev_dbg(hsdma->ddev.dev, "chan 0 no desc to issue\n");
 511        }
 512}
 513
 514static void mtk_hsdma_rx(struct mtk_hsdam_engine *hsdma)
 515{
 516        struct mtk_hsdma_chan *chan;
 517        int next_idx, drx_idx, cnt;
 518
 519        chan = &hsdma->chan[0];
 520        next_idx = HSDMA_NEXT_DESC(chan->rx_idx);
 521        drx_idx = mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX);
 522
 523        cnt = (drx_idx - next_idx) & HSDMA_DESCS_MASK;
 524        if (!cnt)
 525                return;
 526
 527        chan->next_sg += cnt;
 528        chan->rx_idx = (chan->rx_idx + cnt) & HSDMA_DESCS_MASK;
 529
 530        /* update rx crx */
 531        wmb();
 532        mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx);
 533
 534        mtk_hsdma_chan_done(hsdma, chan);
 535}
 536
 537static void mtk_hsdma_tasklet(unsigned long arg)
 538{
 539        struct mtk_hsdam_engine *hsdma = (struct mtk_hsdam_engine *)arg;
 540
 541        mtk_hsdma_rx(hsdma);
 542        mtk_hsdma_tx(hsdma);
 543}
 544
 545static int mtk_hsdam_alloc_desc(struct mtk_hsdam_engine *hsdma,
 546                                struct mtk_hsdma_chan *chan)
 547{
 548        int i;
 549
 550        chan->tx_ring = dma_alloc_coherent(hsdma->ddev.dev,
 551                        2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring),
 552                        &chan->desc_addr, GFP_ATOMIC | __GFP_ZERO);
 553        if (!chan->tx_ring)
 554                goto no_mem;
 555
 556        chan->rx_ring = &chan->tx_ring[HSDMA_DESCS_NUM];
 557
 558        /* init tx ring value */
 559        for (i = 0; i < HSDMA_DESCS_NUM; i++)
 560                chan->tx_ring[i].flags = HSDMA_DESC_LS0 | HSDMA_DESC_DONE;
 561
 562        return 0;
 563no_mem:
 564        return -ENOMEM;
 565}
 566
 567static void mtk_hsdam_free_desc(struct mtk_hsdam_engine *hsdma,
 568                                struct mtk_hsdma_chan *chan)
 569{
 570        if (chan->tx_ring) {
 571                dma_free_coherent(hsdma->ddev.dev,
 572                                2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring),
 573                                chan->tx_ring, chan->desc_addr);
 574                chan->tx_ring = NULL;
 575                chan->rx_ring = NULL;
 576        }
 577}
 578
 579static int mtk_hsdma_init(struct mtk_hsdam_engine *hsdma)
 580{
 581        struct mtk_hsdma_chan *chan;
 582        int ret;
 583        u32 reg;
 584
 585        /* init desc */
 586        chan = &hsdma->chan[0];
 587        ret = mtk_hsdam_alloc_desc(hsdma, chan);
 588        if (ret)
 589                return ret;
 590
 591        /* tx */
 592        mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, chan->desc_addr);
 593        mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, HSDMA_DESCS_NUM);
 594        /* rx */
 595        mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, chan->desc_addr +
 596                        (sizeof(struct hsdma_desc) * HSDMA_DESCS_NUM));
 597        mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, HSDMA_DESCS_NUM);
 598        /* reset */
 599        mtk_hsdma_reset_chan(hsdma, chan);
 600
 601        /* enable rx intr */
 602        mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0);
 603
 604        /* enable dma */
 605        mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT);
 606
 607        /* hardware info */
 608        reg = mtk_hsdma_read(hsdma, HSDMA_REG_INFO);
 609        dev_info(hsdma->ddev.dev, "rx: %d, tx: %d\n",
 610                 (reg >> HSDMA_INFO_RX_SHIFT) & HSDMA_INFO_RX_MASK,
 611                 (reg >> HSDMA_INFO_TX_SHIFT) & HSDMA_INFO_TX_MASK);
 612
 613        hsdma_dump_reg(hsdma);
 614
 615        return ret;
 616}
 617
 618static void mtk_hsdma_uninit(struct mtk_hsdam_engine *hsdma)
 619{
 620        struct mtk_hsdma_chan *chan;
 621
 622        /* disable dma */
 623        mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0);
 624
 625        /* disable intr */
 626        mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0);
 627
 628        /* free desc */
 629        chan = &hsdma->chan[0];
 630        mtk_hsdam_free_desc(hsdma, chan);
 631
 632        /* tx */
 633        mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, 0);
 634        mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, 0);
 635        /* rx */
 636        mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, 0);
 637        mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, 0);
 638        /* reset */
 639        mtk_hsdma_reset_chan(hsdma, chan);
 640}
 641
 642static const struct of_device_id mtk_hsdma_of_match[] = {
 643        { .compatible = "mediatek,mt7621-hsdma" },
 644        { },
 645};
 646
 647static int mtk_hsdma_probe(struct platform_device *pdev)
 648{
 649        const struct of_device_id *match;
 650        struct mtk_hsdma_chan *chan;
 651        struct mtk_hsdam_engine *hsdma;
 652        struct dma_device *dd;
 653        struct resource *res;
 654        int ret;
 655        int irq;
 656        void __iomem *base;
 657
 658        ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 659        if (ret)
 660                return ret;
 661
 662        match = of_match_device(mtk_hsdma_of_match, &pdev->dev);
 663        if (!match)
 664                return -EINVAL;
 665
 666        hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);
 667        if (!hsdma)
 668                return -EINVAL;
 669
 670        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 671        base = devm_ioremap_resource(&pdev->dev, res);
 672        if (IS_ERR(base))
 673                return PTR_ERR(base);
 674        hsdma->base = base + HSDMA_BASE_OFFSET;
 675        tasklet_init(&hsdma->task, mtk_hsdma_tasklet, (unsigned long)hsdma);
 676
 677        irq = platform_get_irq(pdev, 0);
 678        if (irq < 0)
 679                return -EINVAL;
 680        ret = devm_request_irq(&pdev->dev, irq, mtk_hsdma_irq,
 681                               0, dev_name(&pdev->dev), hsdma);
 682        if (ret) {
 683                dev_err(&pdev->dev, "failed to request irq\n");
 684                return ret;
 685        }
 686
 687        device_reset(&pdev->dev);
 688
 689        dd = &hsdma->ddev;
 690        dma_cap_set(DMA_MEMCPY, dd->cap_mask);
 691        dd->copy_align = HSDMA_ALIGN_SIZE;
 692        dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
 693        dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
 694        dd->device_terminate_all = mtk_hsdma_terminate_all;
 695        dd->device_tx_status = mtk_hsdma_tx_status;
 696        dd->device_issue_pending = mtk_hsdma_issue_pending;
 697        dd->dev = &pdev->dev;
 698        dd->dev->dma_parms = &hsdma->dma_parms;
 699        dma_set_max_seg_size(dd->dev, HSDMA_MAX_PLEN);
 700        INIT_LIST_HEAD(&dd->channels);
 701
 702        chan = &hsdma->chan[0];
 703        chan->id = 0;
 704        chan->vchan.desc_free = mtk_hsdma_desc_free;
 705        vchan_init(&chan->vchan, dd);
 706
 707        /* init hardware */
 708        ret = mtk_hsdma_init(hsdma);
 709        if (ret) {
 710                dev_err(&pdev->dev, "failed to alloc ring descs\n");
 711                return ret;
 712        }
 713
 714        ret = dma_async_device_register(dd);
 715        if (ret) {
 716                dev_err(&pdev->dev, "failed to register dma device\n");
 717                return ret;
 718        }
 719
 720        ret = of_dma_controller_register(pdev->dev.of_node,
 721                                         of_dma_xlate_by_chan_id, hsdma);
 722        if (ret) {
 723                dev_err(&pdev->dev, "failed to register of dma controller\n");
 724                goto err_unregister;
 725        }
 726
 727        platform_set_drvdata(pdev, hsdma);
 728
 729        return 0;
 730
 731err_unregister:
 732        dma_async_device_unregister(dd);
 733        return ret;
 734}
 735
 736static int mtk_hsdma_remove(struct platform_device *pdev)
 737{
 738        struct mtk_hsdam_engine *hsdma = platform_get_drvdata(pdev);
 739
 740        mtk_hsdma_uninit(hsdma);
 741
 742        of_dma_controller_free(pdev->dev.of_node);
 743        dma_async_device_unregister(&hsdma->ddev);
 744
 745        return 0;
 746}
 747
 748static struct platform_driver mtk_hsdma_driver = {
 749        .probe = mtk_hsdma_probe,
 750        .remove = mtk_hsdma_remove,
 751        .driver = {
 752                .name = "hsdma-mt7621",
 753                .of_match_table = mtk_hsdma_of_match,
 754        },
 755};
 756module_platform_driver(mtk_hsdma_driver);
 757
 758MODULE_AUTHOR("Michael Lee <igvtee@gmail.com>");
 759MODULE_DESCRIPTION("MTK HSDMA driver");
 760MODULE_LICENSE("GPL v2");
 761