linux/drivers/dma/dma-axi-dmac.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Driver for the Analog Devices AXI-DMAC core
   4 *
   5 * Copyright 2013-2019 Analog Devices Inc.
   6 *  Author: Lars-Peter Clausen <lars@metafoo.de>
   7 */
   8
   9#include <linux/clk.h>
  10#include <linux/device.h>
  11#include <linux/dma-mapping.h>
  12#include <linux/dmaengine.h>
  13#include <linux/err.h>
  14#include <linux/interrupt.h>
  15#include <linux/io.h>
  16#include <linux/kernel.h>
  17#include <linux/module.h>
  18#include <linux/of.h>
  19#include <linux/of_dma.h>
  20#include <linux/platform_device.h>
  21#include <linux/regmap.h>
  22#include <linux/slab.h>
  23#include <linux/fpga/adi-axi-common.h>
  24
  25#include <dt-bindings/dma/axi-dmac.h>
  26
  27#include "dmaengine.h"
  28#include "virt-dma.h"
  29
  30/*
  31 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
  32 * various instantiation parameters which decided the exact feature set support
  33 * by the core.
  34 *
  35 * Each channel of the core has a source interface and a destination interface.
  36 * The number of channels and the type of the channel interfaces is selected at
  37 * configuration time. A interface can either be a connected to a central memory
  38 * interconnect, which allows access to system memory, or it can be connected to
  39 * a dedicated bus which is directly connected to a data port on a peripheral.
  40 * Given that those are configuration options of the core that are selected when
  41 * it is instantiated this means that they can not be changed by software at
  42 * runtime. By extension this means that each channel is uni-directional. It can
  43 * either be device to memory or memory to device, but not both. Also since the
  44 * device side is a dedicated data bus only connected to a single peripheral
  45 * there is no address than can or needs to be configured for the device side.
  46 */
  47
  48#define AXI_DMAC_REG_IRQ_MASK           0x80
  49#define AXI_DMAC_REG_IRQ_PENDING        0x84
  50#define AXI_DMAC_REG_IRQ_SOURCE         0x88
  51
  52#define AXI_DMAC_REG_CTRL               0x400
  53#define AXI_DMAC_REG_TRANSFER_ID        0x404
  54#define AXI_DMAC_REG_START_TRANSFER     0x408
  55#define AXI_DMAC_REG_FLAGS              0x40c
  56#define AXI_DMAC_REG_DEST_ADDRESS       0x410
  57#define AXI_DMAC_REG_SRC_ADDRESS        0x414
  58#define AXI_DMAC_REG_X_LENGTH           0x418
  59#define AXI_DMAC_REG_Y_LENGTH           0x41c
  60#define AXI_DMAC_REG_DEST_STRIDE        0x420
  61#define AXI_DMAC_REG_SRC_STRIDE         0x424
  62#define AXI_DMAC_REG_TRANSFER_DONE      0x428
  63#define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
  64#define AXI_DMAC_REG_STATUS             0x430
  65#define AXI_DMAC_REG_CURRENT_SRC_ADDR   0x434
  66#define AXI_DMAC_REG_CURRENT_DEST_ADDR  0x438
  67#define AXI_DMAC_REG_PARTIAL_XFER_LEN   0x44c
  68#define AXI_DMAC_REG_PARTIAL_XFER_ID    0x450
  69
  70#define AXI_DMAC_CTRL_ENABLE            BIT(0)
  71#define AXI_DMAC_CTRL_PAUSE             BIT(1)
  72
  73#define AXI_DMAC_IRQ_SOT                BIT(0)
  74#define AXI_DMAC_IRQ_EOT                BIT(1)
  75
  76#define AXI_DMAC_FLAG_CYCLIC            BIT(0)
  77#define AXI_DMAC_FLAG_LAST              BIT(1)
  78#define AXI_DMAC_FLAG_PARTIAL_REPORT    BIT(2)
  79
  80#define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
  81
  82/* The maximum ID allocated by the hardware is 31 */
  83#define AXI_DMAC_SG_UNUSED 32U
  84
  85struct axi_dmac_sg {
  86        dma_addr_t src_addr;
  87        dma_addr_t dest_addr;
  88        unsigned int x_len;
  89        unsigned int y_len;
  90        unsigned int dest_stride;
  91        unsigned int src_stride;
  92        unsigned int id;
  93        unsigned int partial_len;
  94        bool schedule_when_free;
  95};
  96
  97struct axi_dmac_desc {
  98        struct virt_dma_desc vdesc;
  99        bool cyclic;
 100        bool have_partial_xfer;
 101
 102        unsigned int num_submitted;
 103        unsigned int num_completed;
 104        unsigned int num_sgs;
 105        struct axi_dmac_sg sg[];
 106};
 107
 108struct axi_dmac_chan {
 109        struct virt_dma_chan vchan;
 110
 111        struct axi_dmac_desc *next_desc;
 112        struct list_head active_descs;
 113        enum dma_transfer_direction direction;
 114
 115        unsigned int src_width;
 116        unsigned int dest_width;
 117        unsigned int src_type;
 118        unsigned int dest_type;
 119
 120        unsigned int max_length;
 121        unsigned int address_align_mask;
 122        unsigned int length_align_mask;
 123
 124        bool hw_partial_xfer;
 125        bool hw_cyclic;
 126        bool hw_2d;
 127};
 128
 129struct axi_dmac {
 130        void __iomem *base;
 131        int irq;
 132
 133        struct clk *clk;
 134
 135        struct dma_device dma_dev;
 136        struct axi_dmac_chan chan;
 137
 138        struct device_dma_parameters dma_parms;
 139};
 140
 141static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)
 142{
 143        return container_of(chan->vchan.chan.device, struct axi_dmac,
 144                dma_dev);
 145}
 146
 147static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c)
 148{
 149        return container_of(c, struct axi_dmac_chan, vchan.chan);
 150}
 151
 152static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc)
 153{
 154        return container_of(vdesc, struct axi_dmac_desc, vdesc);
 155}
 156
 157static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg,
 158        unsigned int val)
 159{
 160        writel(val, axi_dmac->base + reg);
 161}
 162
 163static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg)
 164{
 165        return readl(axi_dmac->base + reg);
 166}
 167
 168static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)
 169{
 170        return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 171}
 172
 173static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
 174{
 175        return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 176}
 177
 178static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
 179{
 180        if (len == 0)
 181                return false;
 182        if ((len & chan->length_align_mask) != 0) /* Not aligned */
 183                return false;
 184        return true;
 185}
 186
 187static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
 188{
 189        if ((addr & chan->address_align_mask) != 0) /* Not aligned */
 190                return false;
 191        return true;
 192}
 193
 194static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 195{
 196        struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 197        struct virt_dma_desc *vdesc;
 198        struct axi_dmac_desc *desc;
 199        struct axi_dmac_sg *sg;
 200        unsigned int flags = 0;
 201        unsigned int val;
 202
 203        val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
 204        if (val) /* Queue is full, wait for the next SOT IRQ */
 205                return;
 206
 207        desc = chan->next_desc;
 208
 209        if (!desc) {
 210                vdesc = vchan_next_desc(&chan->vchan);
 211                if (!vdesc)
 212                        return;
 213                list_move_tail(&vdesc->node, &chan->active_descs);
 214                desc = to_axi_dmac_desc(vdesc);
 215        }
 216        sg = &desc->sg[desc->num_submitted];
 217
 218        /* Already queued in cyclic mode. Wait for it to finish */
 219        if (sg->id != AXI_DMAC_SG_UNUSED) {
 220                sg->schedule_when_free = true;
 221                return;
 222        }
 223
 224        desc->num_submitted++;
 225        if (desc->num_submitted == desc->num_sgs ||
 226            desc->have_partial_xfer) {
 227                if (desc->cyclic)
 228                        desc->num_submitted = 0; /* Start again */
 229                else
 230                        chan->next_desc = NULL;
 231                flags |= AXI_DMAC_FLAG_LAST;
 232        } else {
 233                chan->next_desc = desc;
 234        }
 235
 236        sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
 237
 238        if (axi_dmac_dest_is_mem(chan)) {
 239                axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr);
 240                axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride);
 241        }
 242
 243        if (axi_dmac_src_is_mem(chan)) {
 244                axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr);
 245                axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride);
 246        }
 247
 248        /*
 249         * If the hardware supports cyclic transfers and there is no callback to
 250         * call and only a single segment, enable hw cyclic mode to avoid
 251         * unnecessary interrupts.
 252         */
 253        if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback &&
 254                desc->num_sgs == 1)
 255                flags |= AXI_DMAC_FLAG_CYCLIC;
 256
 257        if (chan->hw_partial_xfer)
 258                flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
 259
 260        axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1);
 261        axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1);
 262        axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
 263        axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
 264}
 265
 266static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
 267{
 268        return list_first_entry_or_null(&chan->active_descs,
 269                struct axi_dmac_desc, vdesc.node);
 270}
 271
 272static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
 273        struct axi_dmac_sg *sg)
 274{
 275        if (chan->hw_2d)
 276                return sg->x_len * sg->y_len;
 277        else
 278                return sg->x_len;
 279}
 280
 281static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan)
 282{
 283        struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 284        struct axi_dmac_desc *desc;
 285        struct axi_dmac_sg *sg;
 286        u32 xfer_done, len, id, i;
 287        bool found_sg;
 288
 289        do {
 290                len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);
 291                id  = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID);
 292
 293                found_sg = false;
 294                list_for_each_entry(desc, &chan->active_descs, vdesc.node) {
 295                        for (i = 0; i < desc->num_sgs; i++) {
 296                                sg = &desc->sg[i];
 297                                if (sg->id == AXI_DMAC_SG_UNUSED)
 298                                        continue;
 299                                if (sg->id == id) {
 300                                        desc->have_partial_xfer = true;
 301                                        sg->partial_len = len;
 302                                        found_sg = true;
 303                                        break;
 304                                }
 305                        }
 306                        if (found_sg)
 307                                break;
 308                }
 309
 310                if (found_sg) {
 311                        dev_dbg(dmac->dma_dev.dev,
 312                                "Found partial segment id=%u, len=%u\n",
 313                                id, len);
 314                } else {
 315                        dev_warn(dmac->dma_dev.dev,
 316                                 "Not found partial segment id=%u, len=%u\n",
 317                                 id, len);
 318                }
 319
 320                /* Check if we have any more partial transfers */
 321                xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 322                xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE);
 323
 324        } while (!xfer_done);
 325}
 326
 327static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
 328        struct axi_dmac_desc *active)
 329{
 330        struct dmaengine_result *rslt = &active->vdesc.tx_result;
 331        unsigned int start = active->num_completed - 1;
 332        struct axi_dmac_sg *sg;
 333        unsigned int i, total;
 334
 335        rslt->result = DMA_TRANS_NOERROR;
 336        rslt->residue = 0;
 337
 338        /*
 339         * We get here if the last completed segment is partial, which
 340         * means we can compute the residue from that segment onwards
 341         */
 342        for (i = start; i < active->num_sgs; i++) {
 343                sg = &active->sg[i];
 344                total = axi_dmac_total_sg_bytes(chan, sg);
 345                rslt->residue += (total - sg->partial_len);
 346        }
 347}
 348
 349static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 350        unsigned int completed_transfers)
 351{
 352        struct axi_dmac_desc *active;
 353        struct axi_dmac_sg *sg;
 354        bool start_next = false;
 355
 356        active = axi_dmac_active_desc(chan);
 357        if (!active)
 358                return false;
 359
 360        if (chan->hw_partial_xfer &&
 361            (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
 362                axi_dmac_dequeue_partial_xfers(chan);
 363
 364        do {
 365                sg = &active->sg[active->num_completed];
 366                if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
 367                        break;
 368                if (!(BIT(sg->id) & completed_transfers))
 369                        break;
 370                active->num_completed++;
 371                sg->id = AXI_DMAC_SG_UNUSED;
 372                if (sg->schedule_when_free) {
 373                        sg->schedule_when_free = false;
 374                        start_next = true;
 375                }
 376
 377                if (sg->partial_len)
 378                        axi_dmac_compute_residue(chan, active);
 379
 380                if (active->cyclic)
 381                        vchan_cyclic_callback(&active->vdesc);
 382
 383                if (active->num_completed == active->num_sgs ||
 384                    sg->partial_len) {
 385                        if (active->cyclic) {
 386                                active->num_completed = 0; /* wrap around */
 387                        } else {
 388                                list_del(&active->vdesc.node);
 389                                vchan_cookie_complete(&active->vdesc);
 390                                active = axi_dmac_active_desc(chan);
 391                        }
 392                }
 393        } while (active);
 394
 395        return start_next;
 396}
 397
 398static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid)
 399{
 400        struct axi_dmac *dmac = devid;
 401        unsigned int pending;
 402        bool start_next = false;
 403
 404        pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING);
 405        if (!pending)
 406                return IRQ_NONE;
 407
 408        axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending);
 409
 410        spin_lock(&dmac->chan.vchan.lock);
 411        /* One or more transfers have finished */
 412        if (pending & AXI_DMAC_IRQ_EOT) {
 413                unsigned int completed;
 414
 415                completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 416                start_next = axi_dmac_transfer_done(&dmac->chan, completed);
 417        }
 418        /* Space has become available in the descriptor queue */
 419        if ((pending & AXI_DMAC_IRQ_SOT) || start_next)
 420                axi_dmac_start_transfer(&dmac->chan);
 421        spin_unlock(&dmac->chan.vchan.lock);
 422
 423        return IRQ_HANDLED;
 424}
 425
 426static int axi_dmac_terminate_all(struct dma_chan *c)
 427{
 428        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 429        struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 430        unsigned long flags;
 431        LIST_HEAD(head);
 432
 433        spin_lock_irqsave(&chan->vchan.lock, flags);
 434        axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
 435        chan->next_desc = NULL;
 436        vchan_get_all_descriptors(&chan->vchan, &head);
 437        list_splice_tail_init(&chan->active_descs, &head);
 438        spin_unlock_irqrestore(&chan->vchan.lock, flags);
 439
 440        vchan_dma_desc_free_list(&chan->vchan, &head);
 441
 442        return 0;
 443}
 444
 445static void axi_dmac_synchronize(struct dma_chan *c)
 446{
 447        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 448
 449        vchan_synchronize(&chan->vchan);
 450}
 451
 452static void axi_dmac_issue_pending(struct dma_chan *c)
 453{
 454        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 455        struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 456        unsigned long flags;
 457
 458        axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE);
 459
 460        spin_lock_irqsave(&chan->vchan.lock, flags);
 461        if (vchan_issue_pending(&chan->vchan))
 462                axi_dmac_start_transfer(chan);
 463        spin_unlock_irqrestore(&chan->vchan.lock, flags);
 464}
 465
 466static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs)
 467{
 468        struct axi_dmac_desc *desc;
 469        unsigned int i;
 470
 471        desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT);
 472        if (!desc)
 473                return NULL;
 474
 475        for (i = 0; i < num_sgs; i++)
 476                desc->sg[i].id = AXI_DMAC_SG_UNUSED;
 477
 478        desc->num_sgs = num_sgs;
 479
 480        return desc;
 481}
 482
 483static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
 484        enum dma_transfer_direction direction, dma_addr_t addr,
 485        unsigned int num_periods, unsigned int period_len,
 486        struct axi_dmac_sg *sg)
 487{
 488        unsigned int num_segments, i;
 489        unsigned int segment_size;
 490        unsigned int len;
 491
 492        /* Split into multiple equally sized segments if necessary */
 493        num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 494        segment_size = DIV_ROUND_UP(period_len, num_segments);
 495        /* Take care of alignment */
 496        segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;
 497
 498        for (i = 0; i < num_periods; i++) {
 499                len = period_len;
 500
 501                while (len > segment_size) {
 502                        if (direction == DMA_DEV_TO_MEM)
 503                                sg->dest_addr = addr;
 504                        else
 505                                sg->src_addr = addr;
 506                        sg->x_len = segment_size;
 507                        sg->y_len = 1;
 508                        sg++;
 509                        addr += segment_size;
 510                        len -= segment_size;
 511                }
 512
 513                if (direction == DMA_DEV_TO_MEM)
 514                        sg->dest_addr = addr;
 515                else
 516                        sg->src_addr = addr;
 517                sg->x_len = len;
 518                sg->y_len = 1;
 519                sg++;
 520                addr += len;
 521        }
 522
 523        return sg;
 524}
 525
 526static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 527        struct dma_chan *c, struct scatterlist *sgl,
 528        unsigned int sg_len, enum dma_transfer_direction direction,
 529        unsigned long flags, void *context)
 530{
 531        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 532        struct axi_dmac_desc *desc;
 533        struct axi_dmac_sg *dsg;
 534        struct scatterlist *sg;
 535        unsigned int num_sgs;
 536        unsigned int i;
 537
 538        if (direction != chan->direction)
 539                return NULL;
 540
 541        num_sgs = 0;
 542        for_each_sg(sgl, sg, sg_len, i)
 543                num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length);
 544
 545        desc = axi_dmac_alloc_desc(num_sgs);
 546        if (!desc)
 547                return NULL;
 548
 549        dsg = desc->sg;
 550
 551        for_each_sg(sgl, sg, sg_len, i) {
 552                if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
 553                    !axi_dmac_check_len(chan, sg_dma_len(sg))) {
 554                        kfree(desc);
 555                        return NULL;
 556                }
 557
 558                dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1,
 559                        sg_dma_len(sg), dsg);
 560        }
 561
 562        desc->cyclic = false;
 563
 564        return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 565}
 566
 567static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
 568        struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
 569        size_t period_len, enum dma_transfer_direction direction,
 570        unsigned long flags)
 571{
 572        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 573        struct axi_dmac_desc *desc;
 574        unsigned int num_periods, num_segments;
 575
 576        if (direction != chan->direction)
 577                return NULL;
 578
 579        if (!axi_dmac_check_len(chan, buf_len) ||
 580            !axi_dmac_check_addr(chan, buf_addr))
 581                return NULL;
 582
 583        if (period_len == 0 || buf_len % period_len)
 584                return NULL;
 585
 586        num_periods = buf_len / period_len;
 587        num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 588
 589        desc = axi_dmac_alloc_desc(num_periods * num_segments);
 590        if (!desc)
 591                return NULL;
 592
 593        axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods,
 594                period_len, desc->sg);
 595
 596        desc->cyclic = true;
 597
 598        return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 599}
 600
 601static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
 602        struct dma_chan *c, struct dma_interleaved_template *xt,
 603        unsigned long flags)
 604{
 605        struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 606        struct axi_dmac_desc *desc;
 607        size_t dst_icg, src_icg;
 608
 609        if (xt->frame_size != 1)
 610                return NULL;
 611
 612        if (xt->dir != chan->direction)
 613                return NULL;
 614
 615        if (axi_dmac_src_is_mem(chan)) {
 616                if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))
 617                        return NULL;
 618        }
 619
 620        if (axi_dmac_dest_is_mem(chan)) {
 621                if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))
 622                        return NULL;
 623        }
 624
 625        dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
 626        src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
 627
 628        if (chan->hw_2d) {
 629                if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
 630                    xt->numf == 0)
 631                        return NULL;
 632                if (xt->sgl[0].size + dst_icg > chan->max_length ||
 633                    xt->sgl[0].size + src_icg > chan->max_length)
 634                        return NULL;
 635        } else {
 636                if (dst_icg != 0 || src_icg != 0)
 637                        return NULL;
 638                if (chan->max_length / xt->sgl[0].size < xt->numf)
 639                        return NULL;
 640                if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))
 641                        return NULL;
 642        }
 643
 644        desc = axi_dmac_alloc_desc(1);
 645        if (!desc)
 646                return NULL;
 647
 648        if (axi_dmac_src_is_mem(chan)) {
 649                desc->sg[0].src_addr = xt->src_start;
 650                desc->sg[0].src_stride = xt->sgl[0].size + src_icg;
 651        }
 652
 653        if (axi_dmac_dest_is_mem(chan)) {
 654                desc->sg[0].dest_addr = xt->dst_start;
 655                desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg;
 656        }
 657
 658        if (chan->hw_2d) {
 659                desc->sg[0].x_len = xt->sgl[0].size;
 660                desc->sg[0].y_len = xt->numf;
 661        } else {
 662                desc->sg[0].x_len = xt->sgl[0].size * xt->numf;
 663                desc->sg[0].y_len = 1;
 664        }
 665
 666        if (flags & DMA_CYCLIC)
 667                desc->cyclic = true;
 668
 669        return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 670}
 671
 672static void axi_dmac_free_chan_resources(struct dma_chan *c)
 673{
 674        vchan_free_chan_resources(to_virt_chan(c));
 675}
 676
 677static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
 678{
 679        kfree(container_of(vdesc, struct axi_dmac_desc, vdesc));
 680}
 681
 682static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
 683{
 684        switch (reg) {
 685        case AXI_DMAC_REG_IRQ_MASK:
 686        case AXI_DMAC_REG_IRQ_SOURCE:
 687        case AXI_DMAC_REG_IRQ_PENDING:
 688        case AXI_DMAC_REG_CTRL:
 689        case AXI_DMAC_REG_TRANSFER_ID:
 690        case AXI_DMAC_REG_START_TRANSFER:
 691        case AXI_DMAC_REG_FLAGS:
 692        case AXI_DMAC_REG_DEST_ADDRESS:
 693        case AXI_DMAC_REG_SRC_ADDRESS:
 694        case AXI_DMAC_REG_X_LENGTH:
 695        case AXI_DMAC_REG_Y_LENGTH:
 696        case AXI_DMAC_REG_DEST_STRIDE:
 697        case AXI_DMAC_REG_SRC_STRIDE:
 698        case AXI_DMAC_REG_TRANSFER_DONE:
 699        case AXI_DMAC_REG_ACTIVE_TRANSFER_ID:
 700        case AXI_DMAC_REG_STATUS:
 701        case AXI_DMAC_REG_CURRENT_SRC_ADDR:
 702        case AXI_DMAC_REG_CURRENT_DEST_ADDR:
 703        case AXI_DMAC_REG_PARTIAL_XFER_LEN:
 704        case AXI_DMAC_REG_PARTIAL_XFER_ID:
 705                return true;
 706        default:
 707                return false;
 708        }
 709}
 710
 711static const struct regmap_config axi_dmac_regmap_config = {
 712        .reg_bits = 32,
 713        .val_bits = 32,
 714        .reg_stride = 4,
 715        .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID,
 716        .readable_reg = axi_dmac_regmap_rdwr,
 717        .writeable_reg = axi_dmac_regmap_rdwr,
 718};
 719
 720/*
 721 * The configuration stored in the devicetree matches the configuration
 722 * parameters of the peripheral instance and allows the driver to know which
 723 * features are implemented and how it should behave.
 724 */
 725static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
 726        struct axi_dmac_chan *chan)
 727{
 728        u32 val;
 729        int ret;
 730
 731        ret = of_property_read_u32(of_chan, "reg", &val);
 732        if (ret)
 733                return ret;
 734
 735        /* We only support 1 channel for now */
 736        if (val != 0)
 737                return -EINVAL;
 738
 739        ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);
 740        if (ret)
 741                return ret;
 742        if (val > AXI_DMAC_BUS_TYPE_FIFO)
 743                return -EINVAL;
 744        chan->src_type = val;
 745
 746        ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);
 747        if (ret)
 748                return ret;
 749        if (val > AXI_DMAC_BUS_TYPE_FIFO)
 750                return -EINVAL;
 751        chan->dest_type = val;
 752
 753        ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);
 754        if (ret)
 755                return ret;
 756        chan->src_width = val / 8;
 757
 758        ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);
 759        if (ret)
 760                return ret;
 761        chan->dest_width = val / 8;
 762
 763        chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;
 764
 765        if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 766                chan->direction = DMA_MEM_TO_MEM;
 767        else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 768                chan->direction = DMA_MEM_TO_DEV;
 769        else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))
 770                chan->direction = DMA_DEV_TO_MEM;
 771        else
 772                chan->direction = DMA_DEV_TO_DEV;
 773
 774        return 0;
 775}
 776
 777static int axi_dmac_detect_caps(struct axi_dmac *dmac)
 778{
 779        struct axi_dmac_chan *chan = &dmac->chan;
 780        unsigned int version;
 781
 782        version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
 783
 784        axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
 785        if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
 786                chan->hw_cyclic = true;
 787
 788        axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
 789        if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
 790                chan->hw_2d = true;
 791
 792        axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
 793        chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
 794        if (chan->max_length != UINT_MAX)
 795                chan->max_length++;
 796
 797        axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff);
 798        if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 &&
 799            chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
 800                dev_err(dmac->dma_dev.dev,
 801                        "Destination memory-mapped interface not supported.");
 802                return -ENODEV;
 803        }
 804
 805        axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff);
 806        if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 &&
 807            chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
 808                dev_err(dmac->dma_dev.dev,
 809                        "Source memory-mapped interface not supported.");
 810                return -ENODEV;
 811        }
 812
 813        if (version >= ADI_AXI_PCORE_VER(4, 2, 'a'))
 814                chan->hw_partial_xfer = true;
 815
 816        if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) {
 817                axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00);
 818                chan->length_align_mask =
 819                        axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
 820        } else {
 821                chan->length_align_mask = chan->address_align_mask;
 822        }
 823
 824        return 0;
 825}
 826
 827static int axi_dmac_probe(struct platform_device *pdev)
 828{
 829        struct device_node *of_channels, *of_chan;
 830        struct dma_device *dma_dev;
 831        struct axi_dmac *dmac;
 832        struct resource *res;
 833        struct regmap *regmap;
 834        int ret;
 835
 836        dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
 837        if (!dmac)
 838                return -ENOMEM;
 839
 840        dmac->irq = platform_get_irq(pdev, 0);
 841        if (dmac->irq < 0)
 842                return dmac->irq;
 843        if (dmac->irq == 0)
 844                return -EINVAL;
 845
 846        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 847        dmac->base = devm_ioremap_resource(&pdev->dev, res);
 848        if (IS_ERR(dmac->base))
 849                return PTR_ERR(dmac->base);
 850
 851        dmac->clk = devm_clk_get(&pdev->dev, NULL);
 852        if (IS_ERR(dmac->clk))
 853                return PTR_ERR(dmac->clk);
 854
 855        INIT_LIST_HEAD(&dmac->chan.active_descs);
 856
 857        of_channels = of_get_child_by_name(pdev->dev.of_node, "adi,channels");
 858        if (of_channels == NULL)
 859                return -ENODEV;
 860
 861        for_each_child_of_node(of_channels, of_chan) {
 862                ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);
 863                if (ret) {
 864                        of_node_put(of_chan);
 865                        of_node_put(of_channels);
 866                        return -EINVAL;
 867                }
 868        }
 869        of_node_put(of_channels);
 870
 871        pdev->dev.dma_parms = &dmac->dma_parms;
 872        dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 873
 874        dma_dev = &dmac->dma_dev;
 875        dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
 876        dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
 877        dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
 878        dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
 879        dma_dev->device_tx_status = dma_cookie_status;
 880        dma_dev->device_issue_pending = axi_dmac_issue_pending;
 881        dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;
 882        dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;
 883        dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;
 884        dma_dev->device_terminate_all = axi_dmac_terminate_all;
 885        dma_dev->device_synchronize = axi_dmac_synchronize;
 886        dma_dev->dev = &pdev->dev;
 887        dma_dev->chancnt = 1;
 888        dma_dev->src_addr_widths = BIT(dmac->chan.src_width);
 889        dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
 890        dma_dev->directions = BIT(dmac->chan.direction);
 891        dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 892        INIT_LIST_HEAD(&dma_dev->channels);
 893
 894        dmac->chan.vchan.desc_free = axi_dmac_desc_free;
 895        vchan_init(&dmac->chan.vchan, dma_dev);
 896
 897        ret = clk_prepare_enable(dmac->clk);
 898        if (ret < 0)
 899                return ret;
 900
 901        ret = axi_dmac_detect_caps(dmac);
 902        if (ret)
 903                goto err_clk_disable;
 904
 905        dma_dev->copy_align = (dmac->chan.address_align_mask + 1);
 906
 907        axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00);
 908
 909        ret = dma_async_device_register(dma_dev);
 910        if (ret)
 911                goto err_clk_disable;
 912
 913        ret = of_dma_controller_register(pdev->dev.of_node,
 914                of_dma_xlate_by_chan_id, dma_dev);
 915        if (ret)
 916                goto err_unregister_device;
 917
 918        ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED,
 919                dev_name(&pdev->dev), dmac);
 920        if (ret)
 921                goto err_unregister_of;
 922
 923        platform_set_drvdata(pdev, dmac);
 924
 925        regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,
 926                 &axi_dmac_regmap_config);
 927        if (IS_ERR(regmap)) {
 928                ret = PTR_ERR(regmap);
 929                goto err_free_irq;
 930        }
 931
 932        return 0;
 933
 934err_free_irq:
 935        free_irq(dmac->irq, dmac);
 936err_unregister_of:
 937        of_dma_controller_free(pdev->dev.of_node);
 938err_unregister_device:
 939        dma_async_device_unregister(&dmac->dma_dev);
 940err_clk_disable:
 941        clk_disable_unprepare(dmac->clk);
 942
 943        return ret;
 944}
 945
 946static int axi_dmac_remove(struct platform_device *pdev)
 947{
 948        struct axi_dmac *dmac = platform_get_drvdata(pdev);
 949
 950        of_dma_controller_free(pdev->dev.of_node);
 951        free_irq(dmac->irq, dmac);
 952        tasklet_kill(&dmac->chan.vchan.task);
 953        dma_async_device_unregister(&dmac->dma_dev);
 954        clk_disable_unprepare(dmac->clk);
 955
 956        return 0;
 957}
 958
 959static const struct of_device_id axi_dmac_of_match_table[] = {
 960        { .compatible = "adi,axi-dmac-1.00.a" },
 961        { },
 962};
 963MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table);
 964
 965static struct platform_driver axi_dmac_driver = {
 966        .driver = {
 967                .name = "dma-axi-dmac",
 968                .of_match_table = axi_dmac_of_match_table,
 969        },
 970        .probe = axi_dmac_probe,
 971        .remove = axi_dmac_remove,
 972};
 973module_platform_driver(axi_dmac_driver);
 974
 975MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 976MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");
 977MODULE_LICENSE("GPL v2");
 978