linux/drivers/dma/tegra20-apb-dma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * DMA driver for Nvidia's Tegra20 APB DMA controller.
   4 *
   5 * Copyright (c) 2012-2013, NVIDIA CORPORATION.  All rights reserved.
   6 */
   7
   8#include <linux/bitops.h>
   9#include <linux/clk.h>
  10#include <linux/delay.h>
  11#include <linux/dmaengine.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/err.h>
  14#include <linux/init.h>
  15#include <linux/interrupt.h>
  16#include <linux/io.h>
  17#include <linux/mm.h>
  18#include <linux/module.h>
  19#include <linux/of.h>
  20#include <linux/of_device.h>
  21#include <linux/of_dma.h>
  22#include <linux/platform_device.h>
  23#include <linux/pm.h>
  24#include <linux/pm_runtime.h>
  25#include <linux/reset.h>
  26#include <linux/slab.h>
  27
  28#include "dmaengine.h"
  29
  30#define CREATE_TRACE_POINTS
  31#include <trace/events/tegra_apb_dma.h>
  32
  33#define TEGRA_APBDMA_GENERAL                    0x0
  34#define TEGRA_APBDMA_GENERAL_ENABLE             BIT(31)
  35
  36#define TEGRA_APBDMA_CONTROL                    0x010
  37#define TEGRA_APBDMA_IRQ_MASK                   0x01c
  38#define TEGRA_APBDMA_IRQ_MASK_SET               0x020
  39
  40/* CSR register */
  41#define TEGRA_APBDMA_CHAN_CSR                   0x00
  42#define TEGRA_APBDMA_CSR_ENB                    BIT(31)
  43#define TEGRA_APBDMA_CSR_IE_EOC                 BIT(30)
  44#define TEGRA_APBDMA_CSR_HOLD                   BIT(29)
  45#define TEGRA_APBDMA_CSR_DIR                    BIT(28)
  46#define TEGRA_APBDMA_CSR_ONCE                   BIT(27)
  47#define TEGRA_APBDMA_CSR_FLOW                   BIT(21)
  48#define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT          16
  49#define TEGRA_APBDMA_CSR_REQ_SEL_MASK           0x1F
  50#define TEGRA_APBDMA_CSR_WCOUNT_MASK            0xFFFC
  51
  52/* STATUS register */
  53#define TEGRA_APBDMA_CHAN_STATUS                0x004
  54#define TEGRA_APBDMA_STATUS_BUSY                BIT(31)
  55#define TEGRA_APBDMA_STATUS_ISE_EOC             BIT(30)
  56#define TEGRA_APBDMA_STATUS_HALT                BIT(29)
  57#define TEGRA_APBDMA_STATUS_PING_PONG           BIT(28)
  58#define TEGRA_APBDMA_STATUS_COUNT_SHIFT         2
  59#define TEGRA_APBDMA_STATUS_COUNT_MASK          0xFFFC
  60
  61#define TEGRA_APBDMA_CHAN_CSRE                  0x00C
  62#define TEGRA_APBDMA_CHAN_CSRE_PAUSE            (1 << 31)
  63
  64/* AHB memory address */
  65#define TEGRA_APBDMA_CHAN_AHBPTR                0x010
  66
  67/* AHB sequence register */
  68#define TEGRA_APBDMA_CHAN_AHBSEQ                0x14
  69#define TEGRA_APBDMA_AHBSEQ_INTR_ENB            BIT(31)
  70#define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8         (0 << 28)
  71#define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16        (1 << 28)
  72#define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32        (2 << 28)
  73#define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64        (3 << 28)
  74#define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128       (4 << 28)
  75#define TEGRA_APBDMA_AHBSEQ_DATA_SWAP           BIT(27)
  76#define TEGRA_APBDMA_AHBSEQ_BURST_1             (4 << 24)
  77#define TEGRA_APBDMA_AHBSEQ_BURST_4             (5 << 24)
  78#define TEGRA_APBDMA_AHBSEQ_BURST_8             (6 << 24)
  79#define TEGRA_APBDMA_AHBSEQ_DBL_BUF             BIT(19)
  80#define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT          16
  81#define TEGRA_APBDMA_AHBSEQ_WRAP_NONE           0
  82
  83/* APB address */
  84#define TEGRA_APBDMA_CHAN_APBPTR                0x018
  85
  86/* APB sequence register */
  87#define TEGRA_APBDMA_CHAN_APBSEQ                0x01c
  88#define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8         (0 << 28)
  89#define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16        (1 << 28)
  90#define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32        (2 << 28)
  91#define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64        (3 << 28)
  92#define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128       (4 << 28)
  93#define TEGRA_APBDMA_APBSEQ_DATA_SWAP           BIT(27)
  94#define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1         (1 << 16)
  95
  96/* Tegra148 specific registers */
  97#define TEGRA_APBDMA_CHAN_WCOUNT                0x20
  98
  99#define TEGRA_APBDMA_CHAN_WORD_TRANSFER         0x24
 100
 101/*
 102 * If any burst is in flight and DMA paused then this is the time to complete
 103 * on-flight burst and update DMA status register.
 104 */
 105#define TEGRA_APBDMA_BURST_COMPLETE_TIME        20
 106
 107/* Channel base address offset from APBDMA base address */
 108#define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET    0x1000
 109
 110#define TEGRA_APBDMA_SLAVE_ID_INVALID   (TEGRA_APBDMA_CSR_REQ_SEL_MASK + 1)
 111
 112struct tegra_dma;
 113
 114/*
 115 * tegra_dma_chip_data Tegra chip specific DMA data
 116 * @nr_channels: Number of channels available in the controller.
 117 * @channel_reg_size: Channel register size/stride.
 118 * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
 119 * @support_channel_pause: Support channel wise pause of dma.
 120 * @support_separate_wcount_reg: Support separate word count register.
 121 */
 122struct tegra_dma_chip_data {
 123        int nr_channels;
 124        int channel_reg_size;
 125        int max_dma_count;
 126        bool support_channel_pause;
 127        bool support_separate_wcount_reg;
 128};
 129
 130/* DMA channel registers */
 131struct tegra_dma_channel_regs {
 132        unsigned long   csr;
 133        unsigned long   ahb_ptr;
 134        unsigned long   apb_ptr;
 135        unsigned long   ahb_seq;
 136        unsigned long   apb_seq;
 137        unsigned long   wcount;
 138};
 139
 140/*
 141 * tegra_dma_sg_req: DMA request details to configure hardware. This
 142 * contains the details for one transfer to configure DMA hw.
 143 * The client's request for data transfer can be broken into multiple
 144 * sub-transfer as per requester details and hw support.
 145 * This sub transfer get added in the list of transfer and point to Tegra
 146 * DMA descriptor which manages the transfer details.
 147 */
 148struct tegra_dma_sg_req {
 149        struct tegra_dma_channel_regs   ch_regs;
 150        unsigned int                    req_len;
 151        bool                            configured;
 152        bool                            last_sg;
 153        struct list_head                node;
 154        struct tegra_dma_desc           *dma_desc;
 155};
 156
 157/*
 158 * tegra_dma_desc: Tegra DMA descriptors which manages the client requests.
 159 * This descriptor keep track of transfer status, callbacks and request
 160 * counts etc.
 161 */
 162struct tegra_dma_desc {
 163        struct dma_async_tx_descriptor  txd;
 164        unsigned int                    bytes_requested;
 165        unsigned int                    bytes_transferred;
 166        enum dma_status                 dma_status;
 167        struct list_head                node;
 168        struct list_head                tx_list;
 169        struct list_head                cb_node;
 170        int                             cb_count;
 171};
 172
 173struct tegra_dma_channel;
 174
 175typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc,
 176                                bool to_terminate);
 177
 178/* tegra_dma_channel: Channel specific information */
 179struct tegra_dma_channel {
 180        struct dma_chan         dma_chan;
 181        char                    name[12];
 182        bool                    config_init;
 183        int                     id;
 184        int                     irq;
 185        void __iomem            *chan_addr;
 186        spinlock_t              lock;
 187        bool                    busy;
 188        struct tegra_dma        *tdma;
 189        bool                    cyclic;
 190
 191        /* Different lists for managing the requests */
 192        struct list_head        free_sg_req;
 193        struct list_head        pending_sg_req;
 194        struct list_head        free_dma_desc;
 195        struct list_head        cb_desc;
 196
 197        /* ISR handler and tasklet for bottom half of isr handling */
 198        dma_isr_handler         isr_handler;
 199        struct tasklet_struct   tasklet;
 200
 201        /* Channel-slave specific configuration */
 202        unsigned int slave_id;
 203        struct dma_slave_config dma_sconfig;
 204        struct tegra_dma_channel_regs   channel_reg;
 205};
 206
 207/* tegra_dma: Tegra DMA specific information */
 208struct tegra_dma {
 209        struct dma_device               dma_dev;
 210        struct device                   *dev;
 211        struct clk                      *dma_clk;
 212        struct reset_control            *rst;
 213        spinlock_t                      global_lock;
 214        void __iomem                    *base_addr;
 215        const struct tegra_dma_chip_data *chip_data;
 216
 217        /*
 218         * Counter for managing global pausing of the DMA controller.
 219         * Only applicable for devices that don't support individual
 220         * channel pausing.
 221         */
 222        u32                             global_pause_count;
 223
 224        /* Some register need to be cache before suspend */
 225        u32                             reg_gen;
 226
 227        /* Last member of the structure */
 228        struct tegra_dma_channel channels[0];
 229};
 230
 231static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)
 232{
 233        writel(val, tdma->base_addr + reg);
 234}
 235
 236static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg)
 237{
 238        return readl(tdma->base_addr + reg);
 239}
 240
 241static inline void tdc_write(struct tegra_dma_channel *tdc,
 242                u32 reg, u32 val)
 243{
 244        writel(val, tdc->chan_addr + reg);
 245}
 246
 247static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
 248{
 249        return readl(tdc->chan_addr + reg);
 250}
 251
 252static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
 253{
 254        return container_of(dc, struct tegra_dma_channel, dma_chan);
 255}
 256
 257static inline struct tegra_dma_desc *txd_to_tegra_dma_desc(
 258                struct dma_async_tx_descriptor *td)
 259{
 260        return container_of(td, struct tegra_dma_desc, txd);
 261}
 262
 263static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
 264{
 265        return &tdc->dma_chan.dev->device;
 266}
 267
 268static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx);
 269static int tegra_dma_runtime_suspend(struct device *dev);
 270static int tegra_dma_runtime_resume(struct device *dev);
 271
 272/* Get DMA desc from free list, if not there then allocate it.  */
 273static struct tegra_dma_desc *tegra_dma_desc_get(
 274                struct tegra_dma_channel *tdc)
 275{
 276        struct tegra_dma_desc *dma_desc;
 277        unsigned long flags;
 278
 279        spin_lock_irqsave(&tdc->lock, flags);
 280
 281        /* Do not allocate if desc are waiting for ack */
 282        list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
 283                if (async_tx_test_ack(&dma_desc->txd)) {
 284                        list_del(&dma_desc->node);
 285                        spin_unlock_irqrestore(&tdc->lock, flags);
 286                        dma_desc->txd.flags = 0;
 287                        return dma_desc;
 288                }
 289        }
 290
 291        spin_unlock_irqrestore(&tdc->lock, flags);
 292
 293        /* Allocate DMA desc */
 294        dma_desc = kzalloc(sizeof(*dma_desc), GFP_NOWAIT);
 295        if (!dma_desc)
 296                return NULL;
 297
 298        dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan);
 299        dma_desc->txd.tx_submit = tegra_dma_tx_submit;
 300        dma_desc->txd.flags = 0;
 301        return dma_desc;
 302}
 303
 304static void tegra_dma_desc_put(struct tegra_dma_channel *tdc,
 305                struct tegra_dma_desc *dma_desc)
 306{
 307        unsigned long flags;
 308
 309        spin_lock_irqsave(&tdc->lock, flags);
 310        if (!list_empty(&dma_desc->tx_list))
 311                list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req);
 312        list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
 313        spin_unlock_irqrestore(&tdc->lock, flags);
 314}
 315
 316static struct tegra_dma_sg_req *tegra_dma_sg_req_get(
 317                struct tegra_dma_channel *tdc)
 318{
 319        struct tegra_dma_sg_req *sg_req = NULL;
 320        unsigned long flags;
 321
 322        spin_lock_irqsave(&tdc->lock, flags);
 323        if (!list_empty(&tdc->free_sg_req)) {
 324                sg_req = list_first_entry(&tdc->free_sg_req,
 325                                        typeof(*sg_req), node);
 326                list_del(&sg_req->node);
 327                spin_unlock_irqrestore(&tdc->lock, flags);
 328                return sg_req;
 329        }
 330        spin_unlock_irqrestore(&tdc->lock, flags);
 331
 332        sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT);
 333
 334        return sg_req;
 335}
 336
 337static int tegra_dma_slave_config(struct dma_chan *dc,
 338                struct dma_slave_config *sconfig)
 339{
 340        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
 341
 342        if (!list_empty(&tdc->pending_sg_req)) {
 343                dev_err(tdc2dev(tdc), "Configuration not allowed\n");
 344                return -EBUSY;
 345        }
 346
 347        memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
 348        if (tdc->slave_id == TEGRA_APBDMA_SLAVE_ID_INVALID &&
 349            sconfig->device_fc) {
 350                if (sconfig->slave_id > TEGRA_APBDMA_CSR_REQ_SEL_MASK)
 351                        return -EINVAL;
 352                tdc->slave_id = sconfig->slave_id;
 353        }
 354        tdc->config_init = true;
 355        return 0;
 356}
 357
 358static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,
 359        bool wait_for_burst_complete)
 360{
 361        struct tegra_dma *tdma = tdc->tdma;
 362
 363        spin_lock(&tdma->global_lock);
 364
 365        if (tdc->tdma->global_pause_count == 0) {
 366                tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0);
 367                if (wait_for_burst_complete)
 368                        udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
 369        }
 370
 371        tdc->tdma->global_pause_count++;
 372
 373        spin_unlock(&tdma->global_lock);
 374}
 375
 376static void tegra_dma_global_resume(struct tegra_dma_channel *tdc)
 377{
 378        struct tegra_dma *tdma = tdc->tdma;
 379
 380        spin_lock(&tdma->global_lock);
 381
 382        if (WARN_ON(tdc->tdma->global_pause_count == 0))
 383                goto out;
 384
 385        if (--tdc->tdma->global_pause_count == 0)
 386                tdma_write(tdma, TEGRA_APBDMA_GENERAL,
 387                           TEGRA_APBDMA_GENERAL_ENABLE);
 388
 389out:
 390        spin_unlock(&tdma->global_lock);
 391}
 392
 393static void tegra_dma_pause(struct tegra_dma_channel *tdc,
 394        bool wait_for_burst_complete)
 395{
 396        struct tegra_dma *tdma = tdc->tdma;
 397
 398        if (tdma->chip_data->support_channel_pause) {
 399                tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE,
 400                                TEGRA_APBDMA_CHAN_CSRE_PAUSE);
 401                if (wait_for_burst_complete)
 402                        udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
 403        } else {
 404                tegra_dma_global_pause(tdc, wait_for_burst_complete);
 405        }
 406}
 407
 408static void tegra_dma_resume(struct tegra_dma_channel *tdc)
 409{
 410        struct tegra_dma *tdma = tdc->tdma;
 411
 412        if (tdma->chip_data->support_channel_pause) {
 413                tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0);
 414        } else {
 415                tegra_dma_global_resume(tdc);
 416        }
 417}
 418
 419static void tegra_dma_stop(struct tegra_dma_channel *tdc)
 420{
 421        u32 csr;
 422        u32 status;
 423
 424        /* Disable interrupts */
 425        csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
 426        csr &= ~TEGRA_APBDMA_CSR_IE_EOC;
 427        tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
 428
 429        /* Disable DMA */
 430        csr &= ~TEGRA_APBDMA_CSR_ENB;
 431        tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
 432
 433        /* Clear interrupt status if it is there */
 434        status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
 435        if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
 436                dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
 437                tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
 438        }
 439        tdc->busy = false;
 440}
 441
 442static void tegra_dma_start(struct tegra_dma_channel *tdc,
 443                struct tegra_dma_sg_req *sg_req)
 444{
 445        struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs;
 446
 447        tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr);
 448        tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq);
 449        tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr);
 450        tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq);
 451        tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr);
 452        if (tdc->tdma->chip_data->support_separate_wcount_reg)
 453                tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount);
 454
 455        /* Start DMA */
 456        tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
 457                                ch_regs->csr | TEGRA_APBDMA_CSR_ENB);
 458}
 459
 460static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,
 461                struct tegra_dma_sg_req *nsg_req)
 462{
 463        unsigned long status;
 464
 465        /*
 466         * The DMA controller reloads the new configuration for next transfer
 467         * after last burst of current transfer completes.
 468         * If there is no IEC status then this makes sure that last burst
 469         * has not be completed. There may be case that last burst is on
 470         * flight and so it can complete but because DMA is paused, it
 471         * will not generates interrupt as well as not reload the new
 472         * configuration.
 473         * If there is already IEC status then interrupt handler need to
 474         * load new configuration.
 475         */
 476        tegra_dma_pause(tdc, false);
 477        status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
 478
 479        /*
 480         * If interrupt is pending then do nothing as the ISR will handle
 481         * the programing for new request.
 482         */
 483        if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
 484                dev_err(tdc2dev(tdc),
 485                        "Skipping new configuration as interrupt is pending\n");
 486                tegra_dma_resume(tdc);
 487                return;
 488        }
 489
 490        /* Safe to program new configuration */
 491        tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr);
 492        tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr);
 493        if (tdc->tdma->chip_data->support_separate_wcount_reg)
 494                tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
 495                                                nsg_req->ch_regs.wcount);
 496        tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
 497                                nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB);
 498        nsg_req->configured = true;
 499
 500        tegra_dma_resume(tdc);
 501}
 502
 503static void tdc_start_head_req(struct tegra_dma_channel *tdc)
 504{
 505        struct tegra_dma_sg_req *sg_req;
 506
 507        if (list_empty(&tdc->pending_sg_req))
 508                return;
 509
 510        sg_req = list_first_entry(&tdc->pending_sg_req,
 511                                        typeof(*sg_req), node);
 512        tegra_dma_start(tdc, sg_req);
 513        sg_req->configured = true;
 514        tdc->busy = true;
 515}
 516
 517static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc)
 518{
 519        struct tegra_dma_sg_req *hsgreq;
 520        struct tegra_dma_sg_req *hnsgreq;
 521
 522        if (list_empty(&tdc->pending_sg_req))
 523                return;
 524
 525        hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
 526        if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) {
 527                hnsgreq = list_first_entry(&hsgreq->node,
 528                                        typeof(*hnsgreq), node);
 529                tegra_dma_configure_for_next(tdc, hnsgreq);
 530        }
 531}
 532
 533static inline int get_current_xferred_count(struct tegra_dma_channel *tdc,
 534        struct tegra_dma_sg_req *sg_req, unsigned long status)
 535{
 536        return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4;
 537}
 538
 539static void tegra_dma_abort_all(struct tegra_dma_channel *tdc)
 540{
 541        struct tegra_dma_sg_req *sgreq;
 542        struct tegra_dma_desc *dma_desc;
 543
 544        while (!list_empty(&tdc->pending_sg_req)) {
 545                sgreq = list_first_entry(&tdc->pending_sg_req,
 546                                                typeof(*sgreq), node);
 547                list_move_tail(&sgreq->node, &tdc->free_sg_req);
 548                if (sgreq->last_sg) {
 549                        dma_desc = sgreq->dma_desc;
 550                        dma_desc->dma_status = DMA_ERROR;
 551                        list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
 552
 553                        /* Add in cb list if it is not there. */
 554                        if (!dma_desc->cb_count)
 555                                list_add_tail(&dma_desc->cb_node,
 556                                                        &tdc->cb_desc);
 557                        dma_desc->cb_count++;
 558                }
 559        }
 560        tdc->isr_handler = NULL;
 561}
 562
 563static bool handle_continuous_head_request(struct tegra_dma_channel *tdc,
 564                struct tegra_dma_sg_req *last_sg_req, bool to_terminate)
 565{
 566        struct tegra_dma_sg_req *hsgreq = NULL;
 567
 568        if (list_empty(&tdc->pending_sg_req)) {
 569                dev_err(tdc2dev(tdc), "DMA is running without req\n");
 570                tegra_dma_stop(tdc);
 571                return false;
 572        }
 573
 574        /*
 575         * Check that head req on list should be in flight.
 576         * If it is not in flight then abort transfer as
 577         * looping of transfer can not continue.
 578         */
 579        hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
 580        if (!hsgreq->configured) {
 581                tegra_dma_stop(tdc);
 582                dev_err(tdc2dev(tdc), "Error in DMA transfer, aborting DMA\n");
 583                tegra_dma_abort_all(tdc);
 584                return false;
 585        }
 586
 587        /* Configure next request */
 588        if (!to_terminate)
 589                tdc_configure_next_head_desc(tdc);
 590        return true;
 591}
 592
 593static void handle_once_dma_done(struct tegra_dma_channel *tdc,
 594        bool to_terminate)
 595{
 596        struct tegra_dma_sg_req *sgreq;
 597        struct tegra_dma_desc *dma_desc;
 598
 599        tdc->busy = false;
 600        sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
 601        dma_desc = sgreq->dma_desc;
 602        dma_desc->bytes_transferred += sgreq->req_len;
 603
 604        list_del(&sgreq->node);
 605        if (sgreq->last_sg) {
 606                dma_desc->dma_status = DMA_COMPLETE;
 607                dma_cookie_complete(&dma_desc->txd);
 608                if (!dma_desc->cb_count)
 609                        list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
 610                dma_desc->cb_count++;
 611                list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
 612        }
 613        list_add_tail(&sgreq->node, &tdc->free_sg_req);
 614
 615        /* Do not start DMA if it is going to be terminate */
 616        if (to_terminate || list_empty(&tdc->pending_sg_req))
 617                return;
 618
 619        tdc_start_head_req(tdc);
 620}
 621
 622static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
 623                bool to_terminate)
 624{
 625        struct tegra_dma_sg_req *sgreq;
 626        struct tegra_dma_desc *dma_desc;
 627        bool st;
 628
 629        sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
 630        dma_desc = sgreq->dma_desc;
 631        /* if we dma for long enough the transfer count will wrap */
 632        dma_desc->bytes_transferred =
 633                (dma_desc->bytes_transferred + sgreq->req_len) %
 634                dma_desc->bytes_requested;
 635
 636        /* Callback need to be call */
 637        if (!dma_desc->cb_count)
 638                list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
 639        dma_desc->cb_count++;
 640
 641        /* If not last req then put at end of pending list */
 642        if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {
 643                list_move_tail(&sgreq->node, &tdc->pending_sg_req);
 644                sgreq->configured = false;
 645                st = handle_continuous_head_request(tdc, sgreq, to_terminate);
 646                if (!st)
 647                        dma_desc->dma_status = DMA_ERROR;
 648        }
 649}
 650
 651static void tegra_dma_tasklet(unsigned long data)
 652{
 653        struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data;
 654        struct dmaengine_desc_callback cb;
 655        struct tegra_dma_desc *dma_desc;
 656        unsigned long flags;
 657        int cb_count;
 658
 659        spin_lock_irqsave(&tdc->lock, flags);
 660        while (!list_empty(&tdc->cb_desc)) {
 661                dma_desc  = list_first_entry(&tdc->cb_desc,
 662                                        typeof(*dma_desc), cb_node);
 663                list_del(&dma_desc->cb_node);
 664                dmaengine_desc_get_callback(&dma_desc->txd, &cb);
 665                cb_count = dma_desc->cb_count;
 666                dma_desc->cb_count = 0;
 667                trace_tegra_dma_complete_cb(&tdc->dma_chan, cb_count,
 668                                            cb.callback);
 669                spin_unlock_irqrestore(&tdc->lock, flags);
 670                while (cb_count--)
 671                        dmaengine_desc_callback_invoke(&cb, NULL);
 672                spin_lock_irqsave(&tdc->lock, flags);
 673        }
 674        spin_unlock_irqrestore(&tdc->lock, flags);
 675}
 676
 677static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
 678{
 679        struct tegra_dma_channel *tdc = dev_id;
 680        unsigned long status;
 681        unsigned long flags;
 682
 683        spin_lock_irqsave(&tdc->lock, flags);
 684
 685        trace_tegra_dma_isr(&tdc->dma_chan, irq);
 686        status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
 687        if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
 688                tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
 689                tdc->isr_handler(tdc, false);
 690                tasklet_schedule(&tdc->tasklet);
 691                spin_unlock_irqrestore(&tdc->lock, flags);
 692                return IRQ_HANDLED;
 693        }
 694
 695        spin_unlock_irqrestore(&tdc->lock, flags);
 696        dev_info(tdc2dev(tdc),
 697                "Interrupt already served status 0x%08lx\n", status);
 698        return IRQ_NONE;
 699}
 700
 701static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd)
 702{
 703        struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd);
 704        struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan);
 705        unsigned long flags;
 706        dma_cookie_t cookie;
 707
 708        spin_lock_irqsave(&tdc->lock, flags);
 709        dma_desc->dma_status = DMA_IN_PROGRESS;
 710        cookie = dma_cookie_assign(&dma_desc->txd);
 711        list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req);
 712        spin_unlock_irqrestore(&tdc->lock, flags);
 713        return cookie;
 714}
 715
 716static void tegra_dma_issue_pending(struct dma_chan *dc)
 717{
 718        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
 719        unsigned long flags;
 720
 721        spin_lock_irqsave(&tdc->lock, flags);
 722        if (list_empty(&tdc->pending_sg_req)) {
 723                dev_err(tdc2dev(tdc), "No DMA request\n");
 724                goto end;
 725        }
 726        if (!tdc->busy) {
 727                tdc_start_head_req(tdc);
 728
 729                /* Continuous single mode: Configure next req */
 730                if (tdc->cyclic) {
 731                        /*
 732                         * Wait for 1 burst time for configure DMA for
 733                         * next transfer.
 734                         */
 735                        udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
 736                        tdc_configure_next_head_desc(tdc);
 737                }
 738        }
 739end:
 740        spin_unlock_irqrestore(&tdc->lock, flags);
 741}
 742
 743static int tegra_dma_terminate_all(struct dma_chan *dc)
 744{
 745        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
 746        struct tegra_dma_sg_req *sgreq;
 747        struct tegra_dma_desc *dma_desc;
 748        unsigned long flags;
 749        unsigned long status;
 750        unsigned long wcount;
 751        bool was_busy;
 752
 753        spin_lock_irqsave(&tdc->lock, flags);
 754        if (list_empty(&tdc->pending_sg_req)) {
 755                spin_unlock_irqrestore(&tdc->lock, flags);
 756                return 0;
 757        }
 758
 759        if (!tdc->busy)
 760                goto skip_dma_stop;
 761
 762        /* Pause DMA before checking the queue status */
 763        tegra_dma_pause(tdc, true);
 764
 765        status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
 766        if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
 767                dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__);
 768                tdc->isr_handler(tdc, true);
 769                status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
 770        }
 771        if (tdc->tdma->chip_data->support_separate_wcount_reg)
 772                wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);
 773        else
 774                wcount = status;
 775
 776        was_busy = tdc->busy;
 777        tegra_dma_stop(tdc);
 778
 779        if (!list_empty(&tdc->pending_sg_req) && was_busy) {
 780                sgreq = list_first_entry(&tdc->pending_sg_req,
 781                                        typeof(*sgreq), node);
 782                sgreq->dma_desc->bytes_transferred +=
 783                                get_current_xferred_count(tdc, sgreq, wcount);
 784        }
 785        tegra_dma_resume(tdc);
 786
 787skip_dma_stop:
 788        tegra_dma_abort_all(tdc);
 789
 790        while (!list_empty(&tdc->cb_desc)) {
 791                dma_desc  = list_first_entry(&tdc->cb_desc,
 792                                        typeof(*dma_desc), cb_node);
 793                list_del(&dma_desc->cb_node);
 794                dma_desc->cb_count = 0;
 795        }
 796        spin_unlock_irqrestore(&tdc->lock, flags);
 797        return 0;
 798}
 799
 800static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
 801        dma_cookie_t cookie, struct dma_tx_state *txstate)
 802{
 803        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
 804        struct tegra_dma_desc *dma_desc;
 805        struct tegra_dma_sg_req *sg_req;
 806        enum dma_status ret;
 807        unsigned long flags;
 808        unsigned int residual;
 809
 810        ret = dma_cookie_status(dc, cookie, txstate);
 811        if (ret == DMA_COMPLETE)
 812                return ret;
 813
 814        spin_lock_irqsave(&tdc->lock, flags);
 815
 816        /* Check on wait_ack desc status */
 817        list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
 818                if (dma_desc->txd.cookie == cookie) {
 819                        ret = dma_desc->dma_status;
 820                        goto found;
 821                }
 822        }
 823
 824        /* Check in pending list */
 825        list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {
 826                dma_desc = sg_req->dma_desc;
 827                if (dma_desc->txd.cookie == cookie) {
 828                        ret = dma_desc->dma_status;
 829                        goto found;
 830                }
 831        }
 832
 833        dev_dbg(tdc2dev(tdc), "cookie %d not found\n", cookie);
 834        dma_desc = NULL;
 835
 836found:
 837        if (dma_desc && txstate) {
 838                residual = dma_desc->bytes_requested -
 839                           (dma_desc->bytes_transferred %
 840                            dma_desc->bytes_requested);
 841                dma_set_residue(txstate, residual);
 842        }
 843
 844        trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate);
 845        spin_unlock_irqrestore(&tdc->lock, flags);
 846        return ret;
 847}
 848
 849static inline int get_bus_width(struct tegra_dma_channel *tdc,
 850                enum dma_slave_buswidth slave_bw)
 851{
 852        switch (slave_bw) {
 853        case DMA_SLAVE_BUSWIDTH_1_BYTE:
 854                return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8;
 855        case DMA_SLAVE_BUSWIDTH_2_BYTES:
 856                return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16;
 857        case DMA_SLAVE_BUSWIDTH_4_BYTES:
 858                return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
 859        case DMA_SLAVE_BUSWIDTH_8_BYTES:
 860                return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64;
 861        default:
 862                dev_warn(tdc2dev(tdc),
 863                        "slave bw is not supported, using 32bits\n");
 864                return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
 865        }
 866}
 867
 868static inline int get_burst_size(struct tegra_dma_channel *tdc,
 869        u32 burst_size, enum dma_slave_buswidth slave_bw, int len)
 870{
 871        int burst_byte;
 872        int burst_ahb_width;
 873
 874        /*
 875         * burst_size from client is in terms of the bus_width.
 876         * convert them into AHB memory width which is 4 byte.
 877         */
 878        burst_byte = burst_size * slave_bw;
 879        burst_ahb_width = burst_byte / 4;
 880
 881        /* If burst size is 0 then calculate the burst size based on length */
 882        if (!burst_ahb_width) {
 883                if (len & 0xF)
 884                        return TEGRA_APBDMA_AHBSEQ_BURST_1;
 885                else if ((len >> 4) & 0x1)
 886                        return TEGRA_APBDMA_AHBSEQ_BURST_4;
 887                else
 888                        return TEGRA_APBDMA_AHBSEQ_BURST_8;
 889        }
 890        if (burst_ahb_width < 4)
 891                return TEGRA_APBDMA_AHBSEQ_BURST_1;
 892        else if (burst_ahb_width < 8)
 893                return TEGRA_APBDMA_AHBSEQ_BURST_4;
 894        else
 895                return TEGRA_APBDMA_AHBSEQ_BURST_8;
 896}
 897
 898static int get_transfer_param(struct tegra_dma_channel *tdc,
 899        enum dma_transfer_direction direction, unsigned long *apb_addr,
 900        unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size,
 901        enum dma_slave_buswidth *slave_bw)
 902{
 903        switch (direction) {
 904        case DMA_MEM_TO_DEV:
 905                *apb_addr = tdc->dma_sconfig.dst_addr;
 906                *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
 907                *burst_size = tdc->dma_sconfig.dst_maxburst;
 908                *slave_bw = tdc->dma_sconfig.dst_addr_width;
 909                *csr = TEGRA_APBDMA_CSR_DIR;
 910                return 0;
 911
 912        case DMA_DEV_TO_MEM:
 913                *apb_addr = tdc->dma_sconfig.src_addr;
 914                *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
 915                *burst_size = tdc->dma_sconfig.src_maxburst;
 916                *slave_bw = tdc->dma_sconfig.src_addr_width;
 917                *csr = 0;
 918                return 0;
 919
 920        default:
 921                dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
 922                return -EINVAL;
 923        }
 924        return -EINVAL;
 925}
 926
 927static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc,
 928        struct tegra_dma_channel_regs *ch_regs, u32 len)
 929{
 930        u32 len_field = (len - 4) & 0xFFFC;
 931
 932        if (tdc->tdma->chip_data->support_separate_wcount_reg)
 933                ch_regs->wcount = len_field;
 934        else
 935                ch_regs->csr |= len_field;
 936}
 937
 938static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
 939        struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len,
 940        enum dma_transfer_direction direction, unsigned long flags,
 941        void *context)
 942{
 943        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
 944        struct tegra_dma_desc *dma_desc;
 945        unsigned int i;
 946        struct scatterlist *sg;
 947        unsigned long csr, ahb_seq, apb_ptr, apb_seq;
 948        struct list_head req_list;
 949        struct tegra_dma_sg_req  *sg_req = NULL;
 950        u32 burst_size;
 951        enum dma_slave_buswidth slave_bw;
 952
 953        if (!tdc->config_init) {
 954                dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
 955                return NULL;
 956        }
 957        if (sg_len < 1) {
 958                dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
 959                return NULL;
 960        }
 961
 962        if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
 963                                &burst_size, &slave_bw) < 0)
 964                return NULL;
 965
 966        INIT_LIST_HEAD(&req_list);
 967
 968        ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
 969        ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
 970                                        TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
 971        ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
 972
 973        csr |= TEGRA_APBDMA_CSR_ONCE;
 974
 975        if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) {
 976                csr |= TEGRA_APBDMA_CSR_FLOW;
 977                csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
 978        }
 979
 980        if (flags & DMA_PREP_INTERRUPT)
 981                csr |= TEGRA_APBDMA_CSR_IE_EOC;
 982
 983        apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
 984
 985        dma_desc = tegra_dma_desc_get(tdc);
 986        if (!dma_desc) {
 987                dev_err(tdc2dev(tdc), "DMA descriptors not available\n");
 988                return NULL;
 989        }
 990        INIT_LIST_HEAD(&dma_desc->tx_list);
 991        INIT_LIST_HEAD(&dma_desc->cb_node);
 992        dma_desc->cb_count = 0;
 993        dma_desc->bytes_requested = 0;
 994        dma_desc->bytes_transferred = 0;
 995        dma_desc->dma_status = DMA_IN_PROGRESS;
 996
 997        /* Make transfer requests */
 998        for_each_sg(sgl, sg, sg_len, i) {
 999                u32 len, mem;
1000
1001                mem = sg_dma_address(sg);
1002                len = sg_dma_len(sg);
1003
1004                if ((len & 3) || (mem & 3) ||
1005                                (len > tdc->tdma->chip_data->max_dma_count)) {
1006                        dev_err(tdc2dev(tdc),
1007                                "DMA length/memory address is not supported\n");
1008                        tegra_dma_desc_put(tdc, dma_desc);
1009                        return NULL;
1010                }
1011
1012                sg_req = tegra_dma_sg_req_get(tdc);
1013                if (!sg_req) {
1014                        dev_err(tdc2dev(tdc), "DMA sg-req not available\n");
1015                        tegra_dma_desc_put(tdc, dma_desc);
1016                        return NULL;
1017                }
1018
1019                ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1020                dma_desc->bytes_requested += len;
1021
1022                sg_req->ch_regs.apb_ptr = apb_ptr;
1023                sg_req->ch_regs.ahb_ptr = mem;
1024                sg_req->ch_regs.csr = csr;
1025                tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len);
1026                sg_req->ch_regs.apb_seq = apb_seq;
1027                sg_req->ch_regs.ahb_seq = ahb_seq;
1028                sg_req->configured = false;
1029                sg_req->last_sg = false;
1030                sg_req->dma_desc = dma_desc;
1031                sg_req->req_len = len;
1032
1033                list_add_tail(&sg_req->node, &dma_desc->tx_list);
1034        }
1035        sg_req->last_sg = true;
1036        if (flags & DMA_CTRL_ACK)
1037                dma_desc->txd.flags = DMA_CTRL_ACK;
1038
1039        /*
1040         * Make sure that mode should not be conflicting with currently
1041         * configured mode.
1042         */
1043        if (!tdc->isr_handler) {
1044                tdc->isr_handler = handle_once_dma_done;
1045                tdc->cyclic = false;
1046        } else {
1047                if (tdc->cyclic) {
1048                        dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n");
1049                        tegra_dma_desc_put(tdc, dma_desc);
1050                        return NULL;
1051                }
1052        }
1053
1054        return &dma_desc->txd;
1055}
1056
1057static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1058        struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1059        size_t period_len, enum dma_transfer_direction direction,
1060        unsigned long flags)
1061{
1062        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1063        struct tegra_dma_desc *dma_desc = NULL;
1064        struct tegra_dma_sg_req *sg_req = NULL;
1065        unsigned long csr, ahb_seq, apb_ptr, apb_seq;
1066        int len;
1067        size_t remain_len;
1068        dma_addr_t mem = buf_addr;
1069        u32 burst_size;
1070        enum dma_slave_buswidth slave_bw;
1071
1072        if (!buf_len || !period_len) {
1073                dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1074                return NULL;
1075        }
1076
1077        if (!tdc->config_init) {
1078                dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1079                return NULL;
1080        }
1081
1082        /*
1083         * We allow to take more number of requests till DMA is
1084         * not started. The driver will loop over all requests.
1085         * Once DMA is started then new requests can be queued only after
1086         * terminating the DMA.
1087         */
1088        if (tdc->busy) {
1089                dev_err(tdc2dev(tdc), "Request not allowed when DMA running\n");
1090                return NULL;
1091        }
1092
1093        /*
1094         * We only support cycle transfer when buf_len is multiple of
1095         * period_len.
1096         */
1097        if (buf_len % period_len) {
1098                dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1099                return NULL;
1100        }
1101
1102        len = period_len;
1103        if ((len & 3) || (buf_addr & 3) ||
1104                        (len > tdc->tdma->chip_data->max_dma_count)) {
1105                dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1106                return NULL;
1107        }
1108
1109        if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
1110                                &burst_size, &slave_bw) < 0)
1111                return NULL;
1112
1113        ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
1114        ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
1115                                        TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
1116        ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
1117
1118        if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) {
1119                csr |= TEGRA_APBDMA_CSR_FLOW;
1120                csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
1121        }
1122
1123        if (flags & DMA_PREP_INTERRUPT)
1124                csr |= TEGRA_APBDMA_CSR_IE_EOC;
1125
1126        apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
1127
1128        dma_desc = tegra_dma_desc_get(tdc);
1129        if (!dma_desc) {
1130                dev_err(tdc2dev(tdc), "not enough descriptors available\n");
1131                return NULL;
1132        }
1133
1134        INIT_LIST_HEAD(&dma_desc->tx_list);
1135        INIT_LIST_HEAD(&dma_desc->cb_node);
1136        dma_desc->cb_count = 0;
1137
1138        dma_desc->bytes_transferred = 0;
1139        dma_desc->bytes_requested = buf_len;
1140        remain_len = buf_len;
1141
1142        /* Split transfer equal to period size */
1143        while (remain_len) {
1144                sg_req = tegra_dma_sg_req_get(tdc);
1145                if (!sg_req) {
1146                        dev_err(tdc2dev(tdc), "DMA sg-req not available\n");
1147                        tegra_dma_desc_put(tdc, dma_desc);
1148                        return NULL;
1149                }
1150
1151                ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1152                sg_req->ch_regs.apb_ptr = apb_ptr;
1153                sg_req->ch_regs.ahb_ptr = mem;
1154                sg_req->ch_regs.csr = csr;
1155                tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len);
1156                sg_req->ch_regs.apb_seq = apb_seq;
1157                sg_req->ch_regs.ahb_seq = ahb_seq;
1158                sg_req->configured = false;
1159                sg_req->last_sg = false;
1160                sg_req->dma_desc = dma_desc;
1161                sg_req->req_len = len;
1162
1163                list_add_tail(&sg_req->node, &dma_desc->tx_list);
1164                remain_len -= len;
1165                mem += len;
1166        }
1167        sg_req->last_sg = true;
1168        if (flags & DMA_CTRL_ACK)
1169                dma_desc->txd.flags = DMA_CTRL_ACK;
1170
1171        /*
1172         * Make sure that mode should not be conflicting with currently
1173         * configured mode.
1174         */
1175        if (!tdc->isr_handler) {
1176                tdc->isr_handler = handle_cont_sngl_cycle_dma_done;
1177                tdc->cyclic = true;
1178        } else {
1179                if (!tdc->cyclic) {
1180                        dev_err(tdc2dev(tdc), "DMA configuration conflict\n");
1181                        tegra_dma_desc_put(tdc, dma_desc);
1182                        return NULL;
1183                }
1184        }
1185
1186        return &dma_desc->txd;
1187}
1188
1189static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1190{
1191        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1192        struct tegra_dma *tdma = tdc->tdma;
1193        int ret;
1194
1195        dma_cookie_init(&tdc->dma_chan);
1196        tdc->config_init = false;
1197
1198        ret = pm_runtime_get_sync(tdma->dev);
1199        if (ret < 0)
1200                return ret;
1201
1202        return 0;
1203}
1204
1205static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1206{
1207        struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1208        struct tegra_dma *tdma = tdc->tdma;
1209        struct tegra_dma_desc *dma_desc;
1210        struct tegra_dma_sg_req *sg_req;
1211        struct list_head dma_desc_list;
1212        struct list_head sg_req_list;
1213        unsigned long flags;
1214
1215        INIT_LIST_HEAD(&dma_desc_list);
1216        INIT_LIST_HEAD(&sg_req_list);
1217
1218        dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1219
1220        if (tdc->busy)
1221                tegra_dma_terminate_all(dc);
1222
1223        spin_lock_irqsave(&tdc->lock, flags);
1224        list_splice_init(&tdc->pending_sg_req, &sg_req_list);
1225        list_splice_init(&tdc->free_sg_req, &sg_req_list);
1226        list_splice_init(&tdc->free_dma_desc, &dma_desc_list);
1227        INIT_LIST_HEAD(&tdc->cb_desc);
1228        tdc->config_init = false;
1229        tdc->isr_handler = NULL;
1230        spin_unlock_irqrestore(&tdc->lock, flags);
1231
1232        while (!list_empty(&dma_desc_list)) {
1233                dma_desc = list_first_entry(&dma_desc_list,
1234                                        typeof(*dma_desc), node);
1235                list_del(&dma_desc->node);
1236                kfree(dma_desc);
1237        }
1238
1239        while (!list_empty(&sg_req_list)) {
1240                sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node);
1241                list_del(&sg_req->node);
1242                kfree(sg_req);
1243        }
1244        pm_runtime_put(tdma->dev);
1245
1246        tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID;
1247}
1248
1249static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
1250                                           struct of_dma *ofdma)
1251{
1252        struct tegra_dma *tdma = ofdma->of_dma_data;
1253        struct dma_chan *chan;
1254        struct tegra_dma_channel *tdc;
1255
1256        if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) {
1257                dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]);
1258                return NULL;
1259        }
1260
1261        chan = dma_get_any_slave_channel(&tdma->dma_dev);
1262        if (!chan)
1263                return NULL;
1264
1265        tdc = to_tegra_dma_chan(chan);
1266        tdc->slave_id = dma_spec->args[0];
1267
1268        return chan;
1269}
1270
1271/* Tegra20 specific DMA controller information */
1272static const struct tegra_dma_chip_data tegra20_dma_chip_data = {
1273        .nr_channels            = 16,
1274        .channel_reg_size       = 0x20,
1275        .max_dma_count          = 1024UL * 64,
1276        .support_channel_pause  = false,
1277        .support_separate_wcount_reg = false,
1278};
1279
1280/* Tegra30 specific DMA controller information */
1281static const struct tegra_dma_chip_data tegra30_dma_chip_data = {
1282        .nr_channels            = 32,
1283        .channel_reg_size       = 0x20,
1284        .max_dma_count          = 1024UL * 64,
1285        .support_channel_pause  = false,
1286        .support_separate_wcount_reg = false,
1287};
1288
1289/* Tegra114 specific DMA controller information */
1290static const struct tegra_dma_chip_data tegra114_dma_chip_data = {
1291        .nr_channels            = 32,
1292        .channel_reg_size       = 0x20,
1293        .max_dma_count          = 1024UL * 64,
1294        .support_channel_pause  = true,
1295        .support_separate_wcount_reg = false,
1296};
1297
1298/* Tegra148 specific DMA controller information */
1299static const struct tegra_dma_chip_data tegra148_dma_chip_data = {
1300        .nr_channels            = 32,
1301        .channel_reg_size       = 0x40,
1302        .max_dma_count          = 1024UL * 64,
1303        .support_channel_pause  = true,
1304        .support_separate_wcount_reg = true,
1305};
1306
1307static int tegra_dma_probe(struct platform_device *pdev)
1308{
1309        struct resource *res;
1310        struct tegra_dma *tdma;
1311        int ret;
1312        int i;
1313        const struct tegra_dma_chip_data *cdata;
1314
1315        cdata = of_device_get_match_data(&pdev->dev);
1316        if (!cdata) {
1317                dev_err(&pdev->dev, "Error: No device match data found\n");
1318                return -ENODEV;
1319        }
1320
1321        tdma = devm_kzalloc(&pdev->dev,
1322                            struct_size(tdma, channels, cdata->nr_channels),
1323                            GFP_KERNEL);
1324        if (!tdma)
1325                return -ENOMEM;
1326
1327        tdma->dev = &pdev->dev;
1328        tdma->chip_data = cdata;
1329        platform_set_drvdata(pdev, tdma);
1330
1331        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1332        tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
1333        if (IS_ERR(tdma->base_addr))
1334                return PTR_ERR(tdma->base_addr);
1335
1336        tdma->dma_clk = devm_clk_get(&pdev->dev, NULL);
1337        if (IS_ERR(tdma->dma_clk)) {
1338                dev_err(&pdev->dev, "Error: Missing controller clock\n");
1339                return PTR_ERR(tdma->dma_clk);
1340        }
1341
1342        tdma->rst = devm_reset_control_get(&pdev->dev, "dma");
1343        if (IS_ERR(tdma->rst)) {
1344                dev_err(&pdev->dev, "Error: Missing reset\n");
1345                return PTR_ERR(tdma->rst);
1346        }
1347
1348        spin_lock_init(&tdma->global_lock);
1349
1350        pm_runtime_enable(&pdev->dev);
1351        if (!pm_runtime_enabled(&pdev->dev))
1352                ret = tegra_dma_runtime_resume(&pdev->dev);
1353        else
1354                ret = pm_runtime_get_sync(&pdev->dev);
1355
1356        if (ret < 0) {
1357                pm_runtime_disable(&pdev->dev);
1358                return ret;
1359        }
1360
1361        /* Reset DMA controller */
1362        reset_control_assert(tdma->rst);
1363        udelay(2);
1364        reset_control_deassert(tdma->rst);
1365
1366        /* Enable global DMA registers */
1367        tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
1368        tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1369        tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
1370
1371        pm_runtime_put(&pdev->dev);
1372
1373        INIT_LIST_HEAD(&tdma->dma_dev.channels);
1374        for (i = 0; i < cdata->nr_channels; i++) {
1375                struct tegra_dma_channel *tdc = &tdma->channels[i];
1376
1377                tdc->chan_addr = tdma->base_addr +
1378                                 TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +
1379                                 (i * cdata->channel_reg_size);
1380
1381                res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
1382                if (!res) {
1383                        ret = -EINVAL;
1384                        dev_err(&pdev->dev, "No irq resource for chan %d\n", i);
1385                        goto err_irq;
1386                }
1387                tdc->irq = res->start;
1388                snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i);
1389                ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
1390                if (ret) {
1391                        dev_err(&pdev->dev,
1392                                "request_irq failed with err %d channel %d\n",
1393                                ret, i);
1394                        goto err_irq;
1395                }
1396
1397                tdc->dma_chan.device = &tdma->dma_dev;
1398                dma_cookie_init(&tdc->dma_chan);
1399                list_add_tail(&tdc->dma_chan.device_node,
1400                                &tdma->dma_dev.channels);
1401                tdc->tdma = tdma;
1402                tdc->id = i;
1403                tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID;
1404
1405                tasklet_init(&tdc->tasklet, tegra_dma_tasklet,
1406                                (unsigned long)tdc);
1407                spin_lock_init(&tdc->lock);
1408
1409                INIT_LIST_HEAD(&tdc->pending_sg_req);
1410                INIT_LIST_HEAD(&tdc->free_sg_req);
1411                INIT_LIST_HEAD(&tdc->free_dma_desc);
1412                INIT_LIST_HEAD(&tdc->cb_desc);
1413        }
1414
1415        dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1416        dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1417        dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1418
1419        tdma->global_pause_count = 0;
1420        tdma->dma_dev.dev = &pdev->dev;
1421        tdma->dma_dev.device_alloc_chan_resources =
1422                                        tegra_dma_alloc_chan_resources;
1423        tdma->dma_dev.device_free_chan_resources =
1424                                        tegra_dma_free_chan_resources;
1425        tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1426        tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1427        tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1428                BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1429                BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1430                BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1431        tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1432                BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1433                BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1434                BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1435        tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1436        /*
1437         * XXX The hardware appears to support
1438         * DMA_RESIDUE_GRANULARITY_BURST-level reporting, but it's
1439         * only used by this driver during tegra_dma_terminate_all()
1440         */
1441        tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1442        tdma->dma_dev.device_config = tegra_dma_slave_config;
1443        tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
1444        tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1445        tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1446
1447        ret = dma_async_device_register(&tdma->dma_dev);
1448        if (ret < 0) {
1449                dev_err(&pdev->dev,
1450                        "Tegra20 APB DMA driver registration failed %d\n", ret);
1451                goto err_irq;
1452        }
1453
1454        ret = of_dma_controller_register(pdev->dev.of_node,
1455                                         tegra_dma_of_xlate, tdma);
1456        if (ret < 0) {
1457                dev_err(&pdev->dev,
1458                        "Tegra20 APB DMA OF registration failed %d\n", ret);
1459                goto err_unregister_dma_dev;
1460        }
1461
1462        dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n",
1463                        cdata->nr_channels);
1464        return 0;
1465
1466err_unregister_dma_dev:
1467        dma_async_device_unregister(&tdma->dma_dev);
1468err_irq:
1469        while (--i >= 0) {
1470                struct tegra_dma_channel *tdc = &tdma->channels[i];
1471
1472                free_irq(tdc->irq, tdc);
1473                tasklet_kill(&tdc->tasklet);
1474        }
1475
1476        pm_runtime_disable(&pdev->dev);
1477        if (!pm_runtime_status_suspended(&pdev->dev))
1478                tegra_dma_runtime_suspend(&pdev->dev);
1479        return ret;
1480}
1481
1482static int tegra_dma_remove(struct platform_device *pdev)
1483{
1484        struct tegra_dma *tdma = platform_get_drvdata(pdev);
1485        int i;
1486        struct tegra_dma_channel *tdc;
1487
1488        dma_async_device_unregister(&tdma->dma_dev);
1489
1490        for (i = 0; i < tdma->chip_data->nr_channels; ++i) {
1491                tdc = &tdma->channels[i];
1492                free_irq(tdc->irq, tdc);
1493                tasklet_kill(&tdc->tasklet);
1494        }
1495
1496        pm_runtime_disable(&pdev->dev);
1497        if (!pm_runtime_status_suspended(&pdev->dev))
1498                tegra_dma_runtime_suspend(&pdev->dev);
1499
1500        return 0;
1501}
1502
1503static int tegra_dma_runtime_suspend(struct device *dev)
1504{
1505        struct tegra_dma *tdma = dev_get_drvdata(dev);
1506        int i;
1507
1508        tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL);
1509        for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1510                struct tegra_dma_channel *tdc = &tdma->channels[i];
1511                struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1512
1513                /* Only save the state of DMA channels that are in use */
1514                if (!tdc->config_init)
1515                        continue;
1516
1517                ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
1518                ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR);
1519                ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR);
1520                ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ);
1521                ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ);
1522                if (tdma->chip_data->support_separate_wcount_reg)
1523                        ch_reg->wcount = tdc_read(tdc,
1524                                                  TEGRA_APBDMA_CHAN_WCOUNT);
1525        }
1526
1527        clk_disable_unprepare(tdma->dma_clk);
1528
1529        return 0;
1530}
1531
1532static int tegra_dma_runtime_resume(struct device *dev)
1533{
1534        struct tegra_dma *tdma = dev_get_drvdata(dev);
1535        int i, ret;
1536
1537        ret = clk_prepare_enable(tdma->dma_clk);
1538        if (ret < 0) {
1539                dev_err(dev, "clk_enable failed: %d\n", ret);
1540                return ret;
1541        }
1542
1543        tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen);
1544        tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1545        tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
1546
1547        for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1548                struct tegra_dma_channel *tdc = &tdma->channels[i];
1549                struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1550
1551                /* Only restore the state of DMA channels that are in use */
1552                if (!tdc->config_init)
1553                        continue;
1554
1555                if (tdma->chip_data->support_separate_wcount_reg)
1556                        tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
1557                                  ch_reg->wcount);
1558                tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq);
1559                tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr);
1560                tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq);
1561                tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr);
1562                tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
1563                        (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB));
1564        }
1565
1566        return 0;
1567}
1568
1569static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1570        SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume,
1571                           NULL)
1572        SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1573                                pm_runtime_force_resume)
1574};
1575
1576static const struct of_device_id tegra_dma_of_match[] = {
1577        {
1578                .compatible = "nvidia,tegra148-apbdma",
1579                .data = &tegra148_dma_chip_data,
1580        }, {
1581                .compatible = "nvidia,tegra114-apbdma",
1582                .data = &tegra114_dma_chip_data,
1583        }, {
1584                .compatible = "nvidia,tegra30-apbdma",
1585                .data = &tegra30_dma_chip_data,
1586        }, {
1587                .compatible = "nvidia,tegra20-apbdma",
1588                .data = &tegra20_dma_chip_data,
1589        }, {
1590        },
1591};
1592MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1593
1594static struct platform_driver tegra_dmac_driver = {
1595        .driver = {
1596                .name   = "tegra-apbdma",
1597                .pm     = &tegra_dma_dev_pm_ops,
1598                .of_match_table = tegra_dma_of_match,
1599        },
1600        .probe          = tegra_dma_probe,
1601        .remove         = tegra_dma_remove,
1602};
1603
1604module_platform_driver(tegra_dmac_driver);
1605
1606MODULE_ALIAS("platform:tegra20-apbdma");
1607MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver");
1608MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
1609MODULE_LICENSE("GPL v2");
1610