linux/drivers/dma/sa11x0-dma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * SA11x0 DMAengine support
   4 *
   5 * Copyright (C) 2012 Russell King
   6 *   Derived in part from arch/arm/mach-sa1100/dma.c,
   7 *   Copyright (C) 2000, 2001 by Nicolas Pitre
   8 */
   9#include <linux/sched.h>
  10#include <linux/device.h>
  11#include <linux/dmaengine.h>
  12#include <linux/init.h>
  13#include <linux/interrupt.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/platform_device.h>
  17#include <linux/slab.h>
  18#include <linux/spinlock.h>
  19
  20#include "virt-dma.h"
  21
  22#define NR_PHY_CHAN     6
  23#define DMA_ALIGN       3
  24#define DMA_MAX_SIZE    0x1fff
  25#define DMA_CHUNK_SIZE  0x1000
  26
  27#define DMA_DDAR        0x00
  28#define DMA_DCSR_S      0x04
  29#define DMA_DCSR_C      0x08
  30#define DMA_DCSR_R      0x0c
  31#define DMA_DBSA        0x10
  32#define DMA_DBTA        0x14
  33#define DMA_DBSB        0x18
  34#define DMA_DBTB        0x1c
  35#define DMA_SIZE        0x20
  36
  37#define DCSR_RUN        (1 << 0)
  38#define DCSR_IE         (1 << 1)
  39#define DCSR_ERROR      (1 << 2)
  40#define DCSR_DONEA      (1 << 3)
  41#define DCSR_STRTA      (1 << 4)
  42#define DCSR_DONEB      (1 << 5)
  43#define DCSR_STRTB      (1 << 6)
  44#define DCSR_BIU        (1 << 7)
  45
  46#define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  47#define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  48#define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  49#define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  50#define DDAR_Ser0UDCTr  (0x0 << 4)
  51#define DDAR_Ser0UDCRc  (0x1 << 4)
  52#define DDAR_Ser1SDLCTr (0x2 << 4)
  53#define DDAR_Ser1SDLCRc (0x3 << 4)
  54#define DDAR_Ser1UARTTr (0x4 << 4)
  55#define DDAR_Ser1UARTRc (0x5 << 4)
  56#define DDAR_Ser2ICPTr  (0x6 << 4)
  57#define DDAR_Ser2ICPRc  (0x7 << 4)
  58#define DDAR_Ser3UARTTr (0x8 << 4)
  59#define DDAR_Ser3UARTRc (0x9 << 4)
  60#define DDAR_Ser4MCP0Tr (0xa << 4)
  61#define DDAR_Ser4MCP0Rc (0xb << 4)
  62#define DDAR_Ser4MCP1Tr (0xc << 4)
  63#define DDAR_Ser4MCP1Rc (0xd << 4)
  64#define DDAR_Ser4SSPTr  (0xe << 4)
  65#define DDAR_Ser4SSPRc  (0xf << 4)
  66
  67struct sa11x0_dma_sg {
  68        u32                     addr;
  69        u32                     len;
  70};
  71
  72struct sa11x0_dma_desc {
  73        struct virt_dma_desc    vd;
  74
  75        u32                     ddar;
  76        size_t                  size;
  77        unsigned                period;
  78        bool                    cyclic;
  79
  80        unsigned                sglen;
  81        struct sa11x0_dma_sg    sg[];
  82};
  83
  84struct sa11x0_dma_phy;
  85
  86struct sa11x0_dma_chan {
  87        struct virt_dma_chan    vc;
  88
  89        /* protected by c->vc.lock */
  90        struct sa11x0_dma_phy   *phy;
  91        enum dma_status         status;
  92
  93        /* protected by d->lock */
  94        struct list_head        node;
  95
  96        u32                     ddar;
  97        const char              *name;
  98};
  99
 100struct sa11x0_dma_phy {
 101        void __iomem            *base;
 102        struct sa11x0_dma_dev   *dev;
 103        unsigned                num;
 104
 105        struct sa11x0_dma_chan  *vchan;
 106
 107        /* Protected by c->vc.lock */
 108        unsigned                sg_load;
 109        struct sa11x0_dma_desc  *txd_load;
 110        unsigned                sg_done;
 111        struct sa11x0_dma_desc  *txd_done;
 112        u32                     dbs[2];
 113        u32                     dbt[2];
 114        u32                     dcsr;
 115};
 116
 117struct sa11x0_dma_dev {
 118        struct dma_device       slave;
 119        void __iomem            *base;
 120        spinlock_t              lock;
 121        struct tasklet_struct   task;
 122        struct list_head        chan_pending;
 123        struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 124};
 125
 126static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 127{
 128        return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 129}
 130
 131static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 132{
 133        return container_of(dmadev, struct sa11x0_dma_dev, slave);
 134}
 135
 136static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 137{
 138        struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 139
 140        return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 141}
 142
 143static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 144{
 145        kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 146}
 147
 148static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 149{
 150        list_del(&txd->vd.node);
 151        p->txd_load = txd;
 152        p->sg_load = 0;
 153
 154        dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 155                p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 156}
 157
 158static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 159        struct sa11x0_dma_chan *c)
 160{
 161        struct sa11x0_dma_desc *txd = p->txd_load;
 162        struct sa11x0_dma_sg *sg;
 163        void __iomem *base = p->base;
 164        unsigned dbsx, dbtx;
 165        u32 dcsr;
 166
 167        if (!txd)
 168                return;
 169
 170        dcsr = readl_relaxed(base + DMA_DCSR_R);
 171
 172        /* Don't try to load the next transfer if both buffers are started */
 173        if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 174                return;
 175
 176        if (p->sg_load == txd->sglen) {
 177                if (!txd->cyclic) {
 178                        struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 179
 180                        /*
 181                         * We have reached the end of the current descriptor.
 182                         * Peek at the next descriptor, and if compatible with
 183                         * the current, start processing it.
 184                         */
 185                        if (txn && txn->ddar == txd->ddar) {
 186                                txd = txn;
 187                                sa11x0_dma_start_desc(p, txn);
 188                        } else {
 189                                p->txd_load = NULL;
 190                                return;
 191                        }
 192                } else {
 193                        /* Cyclic: reset back to beginning */
 194                        p->sg_load = 0;
 195                }
 196        }
 197
 198        sg = &txd->sg[p->sg_load++];
 199
 200        /* Select buffer to load according to channel status */
 201        if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 202            ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 203                dbsx = DMA_DBSA;
 204                dbtx = DMA_DBTA;
 205                dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 206        } else {
 207                dbsx = DMA_DBSB;
 208                dbtx = DMA_DBTB;
 209                dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 210        }
 211
 212        writel_relaxed(sg->addr, base + dbsx);
 213        writel_relaxed(sg->len, base + dbtx);
 214        writel(dcsr, base + DMA_DCSR_S);
 215
 216        dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 217                p->num, dcsr,
 218                'A' + (dbsx == DMA_DBSB), sg->addr,
 219                'A' + (dbtx == DMA_DBTB), sg->len);
 220}
 221
 222static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 223        struct sa11x0_dma_chan *c)
 224{
 225        struct sa11x0_dma_desc *txd = p->txd_done;
 226
 227        if (++p->sg_done == txd->sglen) {
 228                if (!txd->cyclic) {
 229                        vchan_cookie_complete(&txd->vd);
 230
 231                        p->sg_done = 0;
 232                        p->txd_done = p->txd_load;
 233
 234                        if (!p->txd_done)
 235                                tasklet_schedule(&p->dev->task);
 236                } else {
 237                        if ((p->sg_done % txd->period) == 0)
 238                                vchan_cyclic_callback(&txd->vd);
 239
 240                        /* Cyclic: reset back to beginning */
 241                        p->sg_done = 0;
 242                }
 243        }
 244
 245        sa11x0_dma_start_sg(p, c);
 246}
 247
 248static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 249{
 250        struct sa11x0_dma_phy *p = dev_id;
 251        struct sa11x0_dma_dev *d = p->dev;
 252        struct sa11x0_dma_chan *c;
 253        u32 dcsr;
 254
 255        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 256        if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 257                return IRQ_NONE;
 258
 259        /* Clear reported status bits */
 260        writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 261                p->base + DMA_DCSR_C);
 262
 263        dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 264
 265        if (dcsr & DCSR_ERROR) {
 266                dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 267                        p->num, dcsr,
 268                        readl_relaxed(p->base + DMA_DDAR),
 269                        readl_relaxed(p->base + DMA_DBSA),
 270                        readl_relaxed(p->base + DMA_DBTA),
 271                        readl_relaxed(p->base + DMA_DBSB),
 272                        readl_relaxed(p->base + DMA_DBTB));
 273        }
 274
 275        c = p->vchan;
 276        if (c) {
 277                unsigned long flags;
 278
 279                spin_lock_irqsave(&c->vc.lock, flags);
 280                /*
 281                 * Now that we're holding the lock, check that the vchan
 282                 * really is associated with this pchan before touching the
 283                 * hardware.  This should always succeed, because we won't
 284                 * change p->vchan or c->phy while the channel is actively
 285                 * transferring.
 286                 */
 287                if (c->phy == p) {
 288                        if (dcsr & DCSR_DONEA)
 289                                sa11x0_dma_complete(p, c);
 290                        if (dcsr & DCSR_DONEB)
 291                                sa11x0_dma_complete(p, c);
 292                }
 293                spin_unlock_irqrestore(&c->vc.lock, flags);
 294        }
 295
 296        return IRQ_HANDLED;
 297}
 298
 299static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 300{
 301        struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 302
 303        /* If the issued list is empty, we have no further txds to process */
 304        if (txd) {
 305                struct sa11x0_dma_phy *p = c->phy;
 306
 307                sa11x0_dma_start_desc(p, txd);
 308                p->txd_done = txd;
 309                p->sg_done = 0;
 310
 311                /* The channel should not have any transfers started */
 312                WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 313                                      (DCSR_STRTA | DCSR_STRTB));
 314
 315                /* Clear the run and start bits before changing DDAR */
 316                writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 317                               p->base + DMA_DCSR_C);
 318                writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 319
 320                /* Try to start both buffers */
 321                sa11x0_dma_start_sg(p, c);
 322                sa11x0_dma_start_sg(p, c);
 323        }
 324}
 325
 326static void sa11x0_dma_tasklet(struct tasklet_struct *t)
 327{
 328        struct sa11x0_dma_dev *d = from_tasklet(d, t, task);
 329        struct sa11x0_dma_phy *p;
 330        struct sa11x0_dma_chan *c;
 331        unsigned pch, pch_alloc = 0;
 332
 333        dev_dbg(d->slave.dev, "tasklet enter\n");
 334
 335        list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 336                spin_lock_irq(&c->vc.lock);
 337                p = c->phy;
 338                if (p && !p->txd_done) {
 339                        sa11x0_dma_start_txd(c);
 340                        if (!p->txd_done) {
 341                                /* No current txd associated with this channel */
 342                                dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 343
 344                                /* Mark this channel free */
 345                                c->phy = NULL;
 346                                p->vchan = NULL;
 347                        }
 348                }
 349                spin_unlock_irq(&c->vc.lock);
 350        }
 351
 352        spin_lock_irq(&d->lock);
 353        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 354                p = &d->phy[pch];
 355
 356                if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 357                        c = list_first_entry(&d->chan_pending,
 358                                struct sa11x0_dma_chan, node);
 359                        list_del_init(&c->node);
 360
 361                        pch_alloc |= 1 << pch;
 362
 363                        /* Mark this channel allocated */
 364                        p->vchan = c;
 365
 366                        dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 367                }
 368        }
 369        spin_unlock_irq(&d->lock);
 370
 371        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 372                if (pch_alloc & (1 << pch)) {
 373                        p = &d->phy[pch];
 374                        c = p->vchan;
 375
 376                        spin_lock_irq(&c->vc.lock);
 377                        c->phy = p;
 378
 379                        sa11x0_dma_start_txd(c);
 380                        spin_unlock_irq(&c->vc.lock);
 381                }
 382        }
 383
 384        dev_dbg(d->slave.dev, "tasklet exit\n");
 385}
 386
 387
 388static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 389{
 390        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 391        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 392        unsigned long flags;
 393
 394        spin_lock_irqsave(&d->lock, flags);
 395        list_del_init(&c->node);
 396        spin_unlock_irqrestore(&d->lock, flags);
 397
 398        vchan_free_chan_resources(&c->vc);
 399}
 400
 401static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 402{
 403        unsigned reg;
 404        u32 dcsr;
 405
 406        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 407
 408        if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 409            (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 410                reg = DMA_DBSA;
 411        else
 412                reg = DMA_DBSB;
 413
 414        return readl_relaxed(p->base + reg);
 415}
 416
 417static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 418        dma_cookie_t cookie, struct dma_tx_state *state)
 419{
 420        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 421        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 422        struct sa11x0_dma_phy *p;
 423        struct virt_dma_desc *vd;
 424        unsigned long flags;
 425        enum dma_status ret;
 426
 427        ret = dma_cookie_status(&c->vc.chan, cookie, state);
 428        if (ret == DMA_COMPLETE)
 429                return ret;
 430
 431        if (!state)
 432                return c->status;
 433
 434        spin_lock_irqsave(&c->vc.lock, flags);
 435        p = c->phy;
 436
 437        /*
 438         * If the cookie is on our issue queue, then the residue is
 439         * its total size.
 440         */
 441        vd = vchan_find_desc(&c->vc, cookie);
 442        if (vd) {
 443                state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 444        } else if (!p) {
 445                state->residue = 0;
 446        } else {
 447                struct sa11x0_dma_desc *txd;
 448                size_t bytes = 0;
 449
 450                if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 451                        txd = p->txd_done;
 452                else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 453                        txd = p->txd_load;
 454                else
 455                        txd = NULL;
 456
 457                ret = c->status;
 458                if (txd) {
 459                        dma_addr_t addr = sa11x0_dma_pos(p);
 460                        unsigned i;
 461
 462                        dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
 463
 464                        for (i = 0; i < txd->sglen; i++) {
 465                                dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 466                                        i, txd->sg[i].addr, txd->sg[i].len);
 467                                if (addr >= txd->sg[i].addr &&
 468                                    addr < txd->sg[i].addr + txd->sg[i].len) {
 469                                        unsigned len;
 470
 471                                        len = txd->sg[i].len -
 472                                                (addr - txd->sg[i].addr);
 473                                        dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 474                                                i, len);
 475                                        bytes += len;
 476                                        i++;
 477                                        break;
 478                                }
 479                        }
 480                        for (; i < txd->sglen; i++) {
 481                                dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 482                                        i, txd->sg[i].addr, txd->sg[i].len);
 483                                bytes += txd->sg[i].len;
 484                        }
 485                }
 486                state->residue = bytes;
 487        }
 488        spin_unlock_irqrestore(&c->vc.lock, flags);
 489
 490        dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
 491
 492        return ret;
 493}
 494
 495/*
 496 * Move pending txds to the issued list, and re-init pending list.
 497 * If not already pending, add this channel to the list of pending
 498 * channels and trigger the tasklet to run.
 499 */
 500static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 501{
 502        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 503        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 504        unsigned long flags;
 505
 506        spin_lock_irqsave(&c->vc.lock, flags);
 507        if (vchan_issue_pending(&c->vc)) {
 508                if (!c->phy) {
 509                        spin_lock(&d->lock);
 510                        if (list_empty(&c->node)) {
 511                                list_add_tail(&c->node, &d->chan_pending);
 512                                tasklet_schedule(&d->task);
 513                                dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 514                        }
 515                        spin_unlock(&d->lock);
 516                }
 517        } else
 518                dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 519        spin_unlock_irqrestore(&c->vc.lock, flags);
 520}
 521
 522static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 523        struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 524        enum dma_transfer_direction dir, unsigned long flags, void *context)
 525{
 526        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 527        struct sa11x0_dma_desc *txd;
 528        struct scatterlist *sgent;
 529        unsigned i, j = sglen;
 530        size_t size = 0;
 531
 532        /* SA11x0 channels can only operate in their native direction */
 533        if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 534                dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 535                        &c->vc, c->ddar, dir);
 536                return NULL;
 537        }
 538
 539        /* Do not allow zero-sized txds */
 540        if (sglen == 0)
 541                return NULL;
 542
 543        for_each_sg(sg, sgent, sglen, i) {
 544                dma_addr_t addr = sg_dma_address(sgent);
 545                unsigned int len = sg_dma_len(sgent);
 546
 547                if (len > DMA_MAX_SIZE)
 548                        j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 549                if (addr & DMA_ALIGN) {
 550                        dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
 551                                &c->vc, &addr);
 552                        return NULL;
 553                }
 554        }
 555
 556        txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
 557        if (!txd) {
 558                dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 559                return NULL;
 560        }
 561
 562        j = 0;
 563        for_each_sg(sg, sgent, sglen, i) {
 564                dma_addr_t addr = sg_dma_address(sgent);
 565                unsigned len = sg_dma_len(sgent);
 566
 567                size += len;
 568
 569                do {
 570                        unsigned tlen = len;
 571
 572                        /*
 573                         * Check whether the transfer will fit.  If not, try
 574                         * to split the transfer up such that we end up with
 575                         * equal chunks - but make sure that we preserve the
 576                         * alignment.  This avoids small segments.
 577                         */
 578                        if (tlen > DMA_MAX_SIZE) {
 579                                unsigned mult = DIV_ROUND_UP(tlen,
 580                                        DMA_MAX_SIZE & ~DMA_ALIGN);
 581
 582                                tlen = (tlen / mult) & ~DMA_ALIGN;
 583                        }
 584
 585                        txd->sg[j].addr = addr;
 586                        txd->sg[j].len = tlen;
 587
 588                        addr += tlen;
 589                        len -= tlen;
 590                        j++;
 591                } while (len);
 592        }
 593
 594        txd->ddar = c->ddar;
 595        txd->size = size;
 596        txd->sglen = j;
 597
 598        dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
 599                &c->vc, &txd->vd, txd->size, txd->sglen);
 600
 601        return vchan_tx_prep(&c->vc, &txd->vd, flags);
 602}
 603
 604static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 605        struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 606        enum dma_transfer_direction dir, unsigned long flags)
 607{
 608        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 609        struct sa11x0_dma_desc *txd;
 610        unsigned i, j, k, sglen, sgperiod;
 611
 612        /* SA11x0 channels can only operate in their native direction */
 613        if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 614                dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 615                        &c->vc, c->ddar, dir);
 616                return NULL;
 617        }
 618
 619        sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 620        sglen = size * sgperiod / period;
 621
 622        /* Do not allow zero-sized txds */
 623        if (sglen == 0)
 624                return NULL;
 625
 626        txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
 627        if (!txd) {
 628                dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 629                return NULL;
 630        }
 631
 632        for (i = k = 0; i < size / period; i++) {
 633                size_t tlen, len = period;
 634
 635                for (j = 0; j < sgperiod; j++, k++) {
 636                        tlen = len;
 637
 638                        if (tlen > DMA_MAX_SIZE) {
 639                                unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 640                                tlen = (tlen / mult) & ~DMA_ALIGN;
 641                        }
 642
 643                        txd->sg[k].addr = addr;
 644                        txd->sg[k].len = tlen;
 645                        addr += tlen;
 646                        len -= tlen;
 647                }
 648
 649                WARN_ON(len != 0);
 650        }
 651
 652        WARN_ON(k != sglen);
 653
 654        txd->ddar = c->ddar;
 655        txd->size = size;
 656        txd->sglen = sglen;
 657        txd->cyclic = 1;
 658        txd->period = sgperiod;
 659
 660        return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 661}
 662
 663static int sa11x0_dma_device_config(struct dma_chan *chan,
 664                                    struct dma_slave_config *cfg)
 665{
 666        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 667        u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 668        dma_addr_t addr;
 669        enum dma_slave_buswidth width;
 670        u32 maxburst;
 671
 672        if (ddar & DDAR_RW) {
 673                addr = cfg->src_addr;
 674                width = cfg->src_addr_width;
 675                maxburst = cfg->src_maxburst;
 676        } else {
 677                addr = cfg->dst_addr;
 678                width = cfg->dst_addr_width;
 679                maxburst = cfg->dst_maxburst;
 680        }
 681
 682        if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 683             width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 684            (maxburst != 4 && maxburst != 8))
 685                return -EINVAL;
 686
 687        if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 688                ddar |= DDAR_DW;
 689        if (maxburst == 8)
 690                ddar |= DDAR_BS;
 691
 692        dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
 693                &c->vc, &addr, width, maxburst);
 694
 695        c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 696
 697        return 0;
 698}
 699
 700static int sa11x0_dma_device_pause(struct dma_chan *chan)
 701{
 702        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 703        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 704        struct sa11x0_dma_phy *p;
 705        unsigned long flags;
 706
 707        dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 708        spin_lock_irqsave(&c->vc.lock, flags);
 709        if (c->status == DMA_IN_PROGRESS) {
 710                c->status = DMA_PAUSED;
 711
 712                p = c->phy;
 713                if (p) {
 714                        writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 715                } else {
 716                        spin_lock(&d->lock);
 717                        list_del_init(&c->node);
 718                        spin_unlock(&d->lock);
 719                }
 720        }
 721        spin_unlock_irqrestore(&c->vc.lock, flags);
 722
 723        return 0;
 724}
 725
 726static int sa11x0_dma_device_resume(struct dma_chan *chan)
 727{
 728        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 729        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 730        struct sa11x0_dma_phy *p;
 731        unsigned long flags;
 732
 733        dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 734        spin_lock_irqsave(&c->vc.lock, flags);
 735        if (c->status == DMA_PAUSED) {
 736                c->status = DMA_IN_PROGRESS;
 737
 738                p = c->phy;
 739                if (p) {
 740                        writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 741                } else if (!list_empty(&c->vc.desc_issued)) {
 742                        spin_lock(&d->lock);
 743                        list_add_tail(&c->node, &d->chan_pending);
 744                        spin_unlock(&d->lock);
 745                }
 746        }
 747        spin_unlock_irqrestore(&c->vc.lock, flags);
 748
 749        return 0;
 750}
 751
 752static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
 753{
 754        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 755        struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 756        struct sa11x0_dma_phy *p;
 757        LIST_HEAD(head);
 758        unsigned long flags;
 759
 760        dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 761        /* Clear the tx descriptor lists */
 762        spin_lock_irqsave(&c->vc.lock, flags);
 763        vchan_get_all_descriptors(&c->vc, &head);
 764
 765        p = c->phy;
 766        if (p) {
 767                dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 768                /* vchan is assigned to a pchan - stop the channel */
 769                writel(DCSR_RUN | DCSR_IE |
 770                       DCSR_STRTA | DCSR_DONEA |
 771                       DCSR_STRTB | DCSR_DONEB,
 772                       p->base + DMA_DCSR_C);
 773
 774                if (p->txd_load) {
 775                        if (p->txd_load != p->txd_done)
 776                                list_add_tail(&p->txd_load->vd.node, &head);
 777                        p->txd_load = NULL;
 778                }
 779                if (p->txd_done) {
 780                        list_add_tail(&p->txd_done->vd.node, &head);
 781                        p->txd_done = NULL;
 782                }
 783                c->phy = NULL;
 784                spin_lock(&d->lock);
 785                p->vchan = NULL;
 786                spin_unlock(&d->lock);
 787                tasklet_schedule(&d->task);
 788        }
 789        spin_unlock_irqrestore(&c->vc.lock, flags);
 790        vchan_dma_desc_free_list(&c->vc, &head);
 791
 792        return 0;
 793}
 794
 795struct sa11x0_dma_channel_desc {
 796        u32 ddar;
 797        const char *name;
 798};
 799
 800#define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 801static const struct sa11x0_dma_channel_desc chan_desc[] = {
 802        CD(Ser0UDCTr, 0),
 803        CD(Ser0UDCRc, DDAR_RW),
 804        CD(Ser1SDLCTr, 0),
 805        CD(Ser1SDLCRc, DDAR_RW),
 806        CD(Ser1UARTTr, 0),
 807        CD(Ser1UARTRc, DDAR_RW),
 808        CD(Ser2ICPTr, 0),
 809        CD(Ser2ICPRc, DDAR_RW),
 810        CD(Ser3UARTTr, 0),
 811        CD(Ser3UARTRc, DDAR_RW),
 812        CD(Ser4MCP0Tr, 0),
 813        CD(Ser4MCP0Rc, DDAR_RW),
 814        CD(Ser4MCP1Tr, 0),
 815        CD(Ser4MCP1Rc, DDAR_RW),
 816        CD(Ser4SSPTr, 0),
 817        CD(Ser4SSPRc, DDAR_RW),
 818};
 819
 820static const struct dma_slave_map sa11x0_dma_map[] = {
 821        { "sa11x0-ir", "tx", "Ser2ICPTr" },
 822        { "sa11x0-ir", "rx", "Ser2ICPRc" },
 823        { "sa11x0-ssp", "tx", "Ser4SSPTr" },
 824        { "sa11x0-ssp", "rx", "Ser4SSPRc" },
 825};
 826
 827static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
 828{
 829        struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 830        const char *p = param;
 831
 832        return !strcmp(c->name, p);
 833}
 834
 835static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 836        struct device *dev)
 837{
 838        unsigned i;
 839
 840        INIT_LIST_HEAD(&dmadev->channels);
 841        dmadev->dev = dev;
 842        dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 843        dmadev->device_config = sa11x0_dma_device_config;
 844        dmadev->device_pause = sa11x0_dma_device_pause;
 845        dmadev->device_resume = sa11x0_dma_device_resume;
 846        dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
 847        dmadev->device_tx_status = sa11x0_dma_tx_status;
 848        dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 849
 850        for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
 851                struct sa11x0_dma_chan *c;
 852
 853                c = kzalloc(sizeof(*c), GFP_KERNEL);
 854                if (!c) {
 855                        dev_err(dev, "no memory for channel %u\n", i);
 856                        return -ENOMEM;
 857                }
 858
 859                c->status = DMA_IN_PROGRESS;
 860                c->ddar = chan_desc[i].ddar;
 861                c->name = chan_desc[i].name;
 862                INIT_LIST_HEAD(&c->node);
 863
 864                c->vc.desc_free = sa11x0_dma_free_desc;
 865                vchan_init(&c->vc, dmadev);
 866        }
 867
 868        return dma_async_device_register(dmadev);
 869}
 870
 871static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 872        void *data)
 873{
 874        int irq = platform_get_irq(pdev, nr);
 875
 876        if (irq <= 0)
 877                return -ENXIO;
 878
 879        return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 880}
 881
 882static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 883        void *data)
 884{
 885        int irq = platform_get_irq(pdev, nr);
 886        if (irq > 0)
 887                free_irq(irq, data);
 888}
 889
 890static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 891{
 892        struct sa11x0_dma_chan *c, *cn;
 893
 894        list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 895                list_del(&c->vc.chan.device_node);
 896                tasklet_kill(&c->vc.task);
 897                kfree(c);
 898        }
 899}
 900
 901static int sa11x0_dma_probe(struct platform_device *pdev)
 902{
 903        struct sa11x0_dma_dev *d;
 904        struct resource *res;
 905        unsigned i;
 906        int ret;
 907
 908        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 909        if (!res)
 910                return -ENXIO;
 911
 912        d = kzalloc(sizeof(*d), GFP_KERNEL);
 913        if (!d) {
 914                ret = -ENOMEM;
 915                goto err_alloc;
 916        }
 917
 918        spin_lock_init(&d->lock);
 919        INIT_LIST_HEAD(&d->chan_pending);
 920
 921        d->slave.filter.fn = sa11x0_dma_filter_fn;
 922        d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
 923        d->slave.filter.map = sa11x0_dma_map;
 924
 925        d->base = ioremap(res->start, resource_size(res));
 926        if (!d->base) {
 927                ret = -ENOMEM;
 928                goto err_ioremap;
 929        }
 930
 931        tasklet_setup(&d->task, sa11x0_dma_tasklet);
 932
 933        for (i = 0; i < NR_PHY_CHAN; i++) {
 934                struct sa11x0_dma_phy *p = &d->phy[i];
 935
 936                p->dev = d;
 937                p->num = i;
 938                p->base = d->base + i * DMA_SIZE;
 939                writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 940                        DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 941                        p->base + DMA_DCSR_C);
 942                writel_relaxed(0, p->base + DMA_DDAR);
 943
 944                ret = sa11x0_dma_request_irq(pdev, i, p);
 945                if (ret) {
 946                        while (i) {
 947                                i--;
 948                                sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 949                        }
 950                        goto err_irq;
 951                }
 952        }
 953
 954        dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 955        dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 956        d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 957        d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 958        d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 959        d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 960        d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 961                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 962        d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 963                                   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 964        ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 965        if (ret) {
 966                dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 967                        ret);
 968                goto err_slave_reg;
 969        }
 970
 971        platform_set_drvdata(pdev, d);
 972        return 0;
 973
 974 err_slave_reg:
 975        sa11x0_dma_free_channels(&d->slave);
 976        for (i = 0; i < NR_PHY_CHAN; i++)
 977                sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 978 err_irq:
 979        tasklet_kill(&d->task);
 980        iounmap(d->base);
 981 err_ioremap:
 982        kfree(d);
 983 err_alloc:
 984        return ret;
 985}
 986
 987static int sa11x0_dma_remove(struct platform_device *pdev)
 988{
 989        struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 990        unsigned pch;
 991
 992        dma_async_device_unregister(&d->slave);
 993
 994        sa11x0_dma_free_channels(&d->slave);
 995        for (pch = 0; pch < NR_PHY_CHAN; pch++)
 996                sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 997        tasklet_kill(&d->task);
 998        iounmap(d->base);
 999        kfree(d);
1000
1001        return 0;
1002}
1003
1004static int sa11x0_dma_suspend(struct device *dev)
1005{
1006        struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1007        unsigned pch;
1008
1009        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1010                struct sa11x0_dma_phy *p = &d->phy[pch];
1011                u32 dcsr, saved_dcsr;
1012
1013                dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1014                if (dcsr & DCSR_RUN) {
1015                        writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1016                        dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1017                }
1018
1019                saved_dcsr &= DCSR_RUN | DCSR_IE;
1020                if (dcsr & DCSR_BIU) {
1021                        p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1022                        p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1023                        p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1024                        p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1025                        saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1026                                      (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1027                } else {
1028                        p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1029                        p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1030                        p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1031                        p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1032                        saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1033                }
1034                p->dcsr = saved_dcsr;
1035
1036                writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1037        }
1038
1039        return 0;
1040}
1041
1042static int sa11x0_dma_resume(struct device *dev)
1043{
1044        struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1045        unsigned pch;
1046
1047        for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1048                struct sa11x0_dma_phy *p = &d->phy[pch];
1049                struct sa11x0_dma_desc *txd = NULL;
1050                u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1051
1052                WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1053
1054                if (p->txd_done)
1055                        txd = p->txd_done;
1056                else if (p->txd_load)
1057                        txd = p->txd_load;
1058
1059                if (!txd)
1060                        continue;
1061
1062                writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1063
1064                writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1065                writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1066                writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1067                writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1068                writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1069        }
1070
1071        return 0;
1072}
1073
1074static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1075        .suspend_noirq = sa11x0_dma_suspend,
1076        .resume_noirq = sa11x0_dma_resume,
1077        .freeze_noirq = sa11x0_dma_suspend,
1078        .thaw_noirq = sa11x0_dma_resume,
1079        .poweroff_noirq = sa11x0_dma_suspend,
1080        .restore_noirq = sa11x0_dma_resume,
1081};
1082
1083static struct platform_driver sa11x0_dma_driver = {
1084        .driver = {
1085                .name   = "sa11x0-dma",
1086                .pm     = &sa11x0_dma_pm_ops,
1087        },
1088        .probe          = sa11x0_dma_probe,
1089        .remove         = sa11x0_dma_remove,
1090};
1091
1092static int __init sa11x0_dma_init(void)
1093{
1094        return platform_driver_register(&sa11x0_dma_driver);
1095}
1096subsys_initcall(sa11x0_dma_init);
1097
1098static void __exit sa11x0_dma_exit(void)
1099{
1100        platform_driver_unregister(&sa11x0_dma_driver);
1101}
1102module_exit(sa11x0_dma_exit);
1103
1104MODULE_AUTHOR("Russell King");
1105MODULE_DESCRIPTION("SA-11x0 DMA driver");
1106MODULE_LICENSE("GPL v2");
1107MODULE_ALIAS("platform:sa11x0-dma");
1108