// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2013-2014 Renesas Electronics Europe Ltd.
 * Author: Guennadi Liakhovetski <g.liakhovetski@gmx.de>
 */

#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/highmem.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/log2.h>
#include <linux/mmc/host.h>
#include <linux/mmc/mmc.h>
#include <linux/mmc/sd.h>
#include <linux/mmc/sdio.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/pinctrl/consumer.h>
#include <linux/platform_device.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#include <linux/time.h>
#include <linux/virtio.h>
#include <linux/workqueue.h>

#define USDHI6_SD_CMD           0x0000
#define USDHI6_SD_PORT_SEL      0x0004
#define USDHI6_SD_ARG           0x0008
#define USDHI6_SD_STOP          0x0010
#define USDHI6_SD_SECCNT        0x0014
#define USDHI6_SD_RSP10         0x0018
#define USDHI6_SD_RSP32         0x0020
#define USDHI6_SD_RSP54         0x0028
#define USDHI6_SD_RSP76         0x0030
#define USDHI6_SD_INFO1         0x0038
#define USDHI6_SD_INFO2         0x003c
#define USDHI6_SD_INFO1_MASK    0x0040
#define USDHI6_SD_INFO2_MASK    0x0044
#define USDHI6_SD_CLK_CTRL      0x0048
#define USDHI6_SD_SIZE          0x004c
#define USDHI6_SD_OPTION        0x0050
#define USDHI6_SD_ERR_STS1      0x0058
#define USDHI6_SD_ERR_STS2      0x005c
#define USDHI6_SD_BUF0          0x0060
#define USDHI6_SDIO_MODE        0x0068
#define USDHI6_SDIO_INFO1       0x006c
#define USDHI6_SDIO_INFO1_MASK  0x0070
#define USDHI6_CC_EXT_MODE      0x01b0
#define USDHI6_SOFT_RST         0x01c0
#define USDHI6_VERSION          0x01c4
#define USDHI6_HOST_MODE        0x01c8
#define USDHI6_SDIF_MODE        0x01cc

#define USDHI6_SD_CMD_APP               0x0040
#define USDHI6_SD_CMD_MODE_RSP_AUTO     0x0000
#define USDHI6_SD_CMD_MODE_RSP_NONE     0x0300
#define USDHI6_SD_CMD_MODE_RSP_R1       0x0400  /* Also R5, R6, R7 */
#define USDHI6_SD_CMD_MODE_RSP_R1B      0x0500  /* R1b */
#define USDHI6_SD_CMD_MODE_RSP_R2       0x0600
#define USDHI6_SD_CMD_MODE_RSP_R3       0x0700  /* Also R4 */
#define USDHI6_SD_CMD_DATA              0x0800
#define USDHI6_SD_CMD_READ              0x1000
#define USDHI6_SD_CMD_MULTI             0x2000
#define USDHI6_SD_CMD_CMD12_AUTO_OFF    0x4000

#define USDHI6_CC_EXT_MODE_SDRW         BIT(1)

#define USDHI6_SD_INFO1_RSP_END         BIT(0)
#define USDHI6_SD_INFO1_ACCESS_END      BIT(2)
#define USDHI6_SD_INFO1_CARD_OUT        BIT(3)
#define USDHI6_SD_INFO1_CARD_IN         BIT(4)
#define USDHI6_SD_INFO1_CD              BIT(5)
#define USDHI6_SD_INFO1_WP              BIT(7)
#define USDHI6_SD_INFO1_D3_CARD_OUT     BIT(8)
#define USDHI6_SD_INFO1_D3_CARD_IN      BIT(9)

#define USDHI6_SD_INFO2_CMD_ERR         BIT(0)
#define USDHI6_SD_INFO2_CRC_ERR         BIT(1)
#define USDHI6_SD_INFO2_END_ERR         BIT(2)
#define USDHI6_SD_INFO2_TOUT            BIT(3)
#define USDHI6_SD_INFO2_IWA_ERR         BIT(4)
#define USDHI6_SD_INFO2_IRA_ERR         BIT(5)
#define USDHI6_SD_INFO2_RSP_TOUT        BIT(6)
#define USDHI6_SD_INFO2_SDDAT0          BIT(7)
#define USDHI6_SD_INFO2_BRE             BIT(8)
#define USDHI6_SD_INFO2_BWE             BIT(9)
#define USDHI6_SD_INFO2_SCLKDIVEN       BIT(13)
#define USDHI6_SD_INFO2_CBSY            BIT(14)
#define USDHI6_SD_INFO2_ILA             BIT(15)

#define USDHI6_SD_INFO1_CARD_INSERT (USDHI6_SD_INFO1_CARD_IN | USDHI6_SD_INFO1_D3_CARD_IN)
#define USDHI6_SD_INFO1_CARD_EJECT (USDHI6_SD_INFO1_CARD_OUT | USDHI6_SD_INFO1_D3_CARD_OUT)
#define USDHI6_SD_INFO1_CARD (USDHI6_SD_INFO1_CARD_INSERT | USDHI6_SD_INFO1_CARD_EJECT)
#define USDHI6_SD_INFO1_CARD_CD (USDHI6_SD_INFO1_CARD_IN | USDHI6_SD_INFO1_CARD_OUT)

#define USDHI6_SD_INFO2_ERR     (USDHI6_SD_INFO2_CMD_ERR |      \
        USDHI6_SD_INFO2_CRC_ERR | USDHI6_SD_INFO2_END_ERR |     \
        USDHI6_SD_INFO2_TOUT | USDHI6_SD_INFO2_IWA_ERR |        \
        USDHI6_SD_INFO2_IRA_ERR | USDHI6_SD_INFO2_RSP_TOUT |    \
        USDHI6_SD_INFO2_ILA)

#define USDHI6_SD_INFO1_IRQ     (USDHI6_SD_INFO1_RSP_END | USDHI6_SD_INFO1_ACCESS_END | \
                                 USDHI6_SD_INFO1_CARD)

#define USDHI6_SD_INFO2_IRQ     (USDHI6_SD_INFO2_ERR | USDHI6_SD_INFO2_BRE | \
                                 USDHI6_SD_INFO2_BWE | 0x0800 | USDHI6_SD_INFO2_ILA)

#define USDHI6_SD_CLK_CTRL_SCLKEN       BIT(8)

#define USDHI6_SD_STOP_STP              BIT(0)
#define USDHI6_SD_STOP_SEC              BIT(8)

#define USDHI6_SDIO_INFO1_IOIRQ         BIT(0)
#define USDHI6_SDIO_INFO1_EXPUB52       BIT(14)
#define USDHI6_SDIO_INFO1_EXWT          BIT(15)

#define USDHI6_SD_ERR_STS1_CRC_NO_ERROR BIT(13)

#define USDHI6_SOFT_RST_RESERVED        (BIT(1) | BIT(2))
#define USDHI6_SOFT_RST_RESET           BIT(0)

#define USDHI6_SD_OPTION_TIMEOUT_SHIFT  4
#define USDHI6_SD_OPTION_TIMEOUT_MASK   (0xf << USDHI6_SD_OPTION_TIMEOUT_SHIFT)
#define USDHI6_SD_OPTION_WIDTH_1        BIT(15)

#define USDHI6_SD_PORT_SEL_PORTS_SHIFT  8

#define USDHI6_SD_CLK_CTRL_DIV_MASK     0xff

#define USDHI6_SDIO_INFO1_IRQ   (USDHI6_SDIO_INFO1_IOIRQ | 3 | \
                                 USDHI6_SDIO_INFO1_EXPUB52 | USDHI6_SDIO_INFO1_EXWT)

#define USDHI6_MIN_DMA 64

#define USDHI6_REQ_TIMEOUT_MS 4000

enum usdhi6_wait_for {
        USDHI6_WAIT_FOR_REQUEST,
        USDHI6_WAIT_FOR_CMD,
        USDHI6_WAIT_FOR_MREAD,
        USDHI6_WAIT_FOR_MWRITE,
        USDHI6_WAIT_FOR_READ,
        USDHI6_WAIT_FOR_WRITE,
        USDHI6_WAIT_FOR_DATA_END,
        USDHI6_WAIT_FOR_STOP,
        USDHI6_WAIT_FOR_DMA,
};

struct usdhi6_page {
        struct page *page;
        void *mapped;           /* mapped page */
};

struct usdhi6_host {
        struct mmc_host *mmc;
        struct mmc_request *mrq;
        void __iomem *base;
        struct clk *clk;

        /* SG memory handling */

        /* Common for multiple and single block requests */
        struct usdhi6_page pg;  /* current page from an SG */
        void *blk_page;         /* either a mapped page, or the bounce buffer */
        size_t offset;          /* offset within a page, including sg->offset */

        /* Blocks, crossing a page boundary */
        size_t head_len;
        struct usdhi6_page head_pg;

        /* A bounce buffer for unaligned blocks or blocks, crossing a page boundary */
        struct scatterlist bounce_sg;
        u8 bounce_buf[512];

        /* Multiple block requests only */
        struct scatterlist *sg; /* current SG segment */
        int page_idx;           /* page index within an SG segment */

        enum usdhi6_wait_for wait;
        u32 status_mask;
        u32 status2_mask;
        u32 sdio_mask;
        u32 io_error;
        u32 irq_status;
        unsigned long imclk;
        unsigned long rate;
        bool app_cmd;

        /* Timeout handling */
        struct delayed_work timeout_work;
        unsigned long timeout;

        /* DMA support */
        struct dma_chan *chan_rx;
        struct dma_chan *chan_tx;
        bool dma_active;

        /* Pin control */
        struct pinctrl *pinctrl;
        struct pinctrl_state *pins_uhs;
};

/*                      I/O primitives                                  */

static void usdhi6_write(struct usdhi6_host *host, u32 reg, u32 data)
{
        iowrite32(data, host->base + reg);
        dev_vdbg(mmc_dev(host->mmc), "%s(0x%p + 0x%x) = 0x%x\n", __func__,
                host->base, reg, data);
}

static void usdhi6_write16(struct usdhi6_host *host, u32 reg, u16 data)
{
        iowrite16(data, host->base + reg);
        dev_vdbg(mmc_dev(host->mmc), "%s(0x%p + 0x%x) = 0x%x\n", __func__,
                host->base, reg, data);
}

static u32 usdhi6_read(struct usdhi6_host *host, u32 reg)
{
        u32 data = ioread32(host->base + reg);
        dev_vdbg(mmc_dev(host->mmc), "%s(0x%p + 0x%x) = 0x%x\n", __func__,
                host->base, reg, data);
        return data;
}

static u16 usdhi6_read16(struct usdhi6_host *host, u32 reg)
{
        u16 data = ioread16(host->base + reg);
        dev_vdbg(mmc_dev(host->mmc), "%s(0x%p + 0x%x) = 0x%x\n", __func__,
                host->base, reg, data);
        return data;
}

static void usdhi6_irq_enable(struct usdhi6_host *host, u32 info1, u32 info2)
{
        host->status_mask = USDHI6_SD_INFO1_IRQ & ~info1;
        host->status2_mask = USDHI6_SD_INFO2_IRQ & ~info2;
        usdhi6_write(host, USDHI6_SD_INFO1_MASK, host->status_mask);
        usdhi6_write(host, USDHI6_SD_INFO2_MASK, host->status2_mask);
}

static void usdhi6_wait_for_resp(struct usdhi6_host *host)
{
        usdhi6_irq_enable(host, USDHI6_SD_INFO1_RSP_END |
                          USDHI6_SD_INFO1_ACCESS_END | USDHI6_SD_INFO1_CARD_CD,
                          USDHI6_SD_INFO2_ERR);
}

static void usdhi6_wait_for_brwe(struct usdhi6_host *host, bool read)
{
        usdhi6_irq_enable(host, USDHI6_SD_INFO1_ACCESS_END |
                          USDHI6_SD_INFO1_CARD_CD, USDHI6_SD_INFO2_ERR |
                          (read ? USDHI6_SD_INFO2_BRE : USDHI6_SD_INFO2_BWE));
}

static void usdhi6_only_cd(struct usdhi6_host *host)
{
        /* Mask all except card hotplug */
        usdhi6_irq_enable(host, USDHI6_SD_INFO1_CARD_CD, 0);
}

static void usdhi6_mask_all(struct usdhi6_host *host)
{
        usdhi6_irq_enable(host, 0, 0);
}

static int usdhi6_error_code(struct usdhi6_host *host)
{
        u32 err;

        usdhi6_write(host, USDHI6_SD_STOP, USDHI6_SD_STOP_STP);

        if (host->io_error &
            (USDHI6_SD_INFO2_RSP_TOUT | USDHI6_SD_INFO2_TOUT)) {
                u32 rsp54 = usdhi6_read(host, USDHI6_SD_RSP54);
                int opc = host->mrq ? host->mrq->cmd->opcode : -1;

                err = usdhi6_read(host, USDHI6_SD_ERR_STS2);
                /* Response timeout is often normal, don't spam the log */
                if (host->wait == USDHI6_WAIT_FOR_CMD)
                        dev_dbg(mmc_dev(host->mmc),
                                "T-out sts 0x%x, resp 0x%x, state %u, CMD%d\n",
                                err, rsp54, host->wait, opc);
                else
                        dev_warn(mmc_dev(host->mmc),
                                 "T-out sts 0x%x, resp 0x%x, state %u, CMD%d\n",
                                 err, rsp54, host->wait, opc);
                return -ETIMEDOUT;
        }

        err = usdhi6_read(host, USDHI6_SD_ERR_STS1);
        if (err != USDHI6_SD_ERR_STS1_CRC_NO_ERROR)
                dev_warn(mmc_dev(host->mmc), "Err sts 0x%x, state %u, CMD%d\n",
                         err, host->wait, host->mrq ? host->mrq->cmd->opcode : -1);
        if (host->io_error & USDHI6_SD_INFO2_ILA)
                return -EILSEQ;

        return -EIO;
}

/*                      Scatter-Gather management                       */

/*
 * In PIO mode we have to map each page separately, using kmap(). That way
 * adjacent pages are mapped to non-adjacent virtual addresses. That's why we
 * have to use a bounce buffer for blocks, crossing page boundaries. Such blocks
 * have been observed with an SDIO WiFi card (b43 driver).
 */
static void usdhi6_blk_bounce(struct usdhi6_host *host,
                              struct scatterlist *sg)
{
        struct mmc_data *data = host->mrq->data;
        size_t blk_head = host->head_len;

        dev_dbg(mmc_dev(host->mmc), "%s(): CMD%u of %u SG: %ux%u @ 0x%x\n",
                __func__, host->mrq->cmd->opcode, data->sg_len,
                data->blksz, data->blocks, sg->offset);

        host->head_pg.page      = host->pg.page;
        host->head_pg.mapped    = host->pg.mapped;
        host->pg.page           = nth_page(host->pg.page, 1);
        host->pg.mapped         = kmap(host->pg.page);

        host->blk_page = host->bounce_buf;
        host->offset = 0;

        if (data->flags & MMC_DATA_READ)
                return;

        memcpy(host->bounce_buf, host->head_pg.mapped + PAGE_SIZE - blk_head,
               blk_head);
        memcpy(host->bounce_buf + blk_head, host->pg.mapped,
               data->blksz - blk_head);
}

/* Only called for multiple block IO */
static void usdhi6_sg_prep(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        struct mmc_data *data = mrq->data;

        usdhi6_write(host, USDHI6_SD_SECCNT, data->blocks);

        host->sg = data->sg;
        /* TODO: if we always map, this is redundant */
        host->offset = host->sg->offset;
}

/* Map the first page in an SG segment: common for multiple and single block IO */
static void *usdhi6_sg_map(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;
        struct scatterlist *sg = data->sg_len > 1 ? host->sg : data->sg;
        size_t head = PAGE_SIZE - sg->offset;
        size_t blk_head = head % data->blksz;

        WARN(host->pg.page, "%p not properly unmapped!\n", host->pg.page);
        if (WARN(sg_dma_len(sg) % data->blksz,
                 "SG size %u isn't a multiple of block size %u\n",
                 sg_dma_len(sg), data->blksz))
                return NULL;

        host->pg.page = sg_page(sg);
        host->pg.mapped = kmap(host->pg.page);
        host->offset = sg->offset;

        /*
         * Block size must be a power of 2 for multi-block transfers,
         * therefore blk_head is equal for all pages in this SG
         */
        host->head_len = blk_head;

        if (head < data->blksz)
                /*
                 * The first block in the SG crosses a page boundary.
                 * Max blksz = 512, so blocks can only span 2 pages
                 */
                usdhi6_blk_bounce(host, sg);
        else
                host->blk_page = host->pg.mapped;

        dev_dbg(mmc_dev(host->mmc), "Mapped %p (%lx) at %p + %u for CMD%u @ 0x%p\n",
                host->pg.page, page_to_pfn(host->pg.page), host->pg.mapped,
                sg->offset, host->mrq->cmd->opcode, host->mrq);

        return host->blk_page + host->offset;
}

/* Unmap the current page: common for multiple and single block IO */
static void usdhi6_sg_unmap(struct usdhi6_host *host, bool force)
{
        struct mmc_data *data = host->mrq->data;
        struct page *page = host->head_pg.page;

        if (page) {
                /* Previous block was cross-page boundary */
                struct scatterlist *sg = data->sg_len > 1 ?
                        host->sg : data->sg;
                size_t blk_head = host->head_len;

                if (!data->error && data->flags & MMC_DATA_READ) {
                        memcpy(host->head_pg.mapped + PAGE_SIZE - blk_head,
                               host->bounce_buf, blk_head);
                        memcpy(host->pg.mapped, host->bounce_buf + blk_head,
                               data->blksz - blk_head);
                }

                flush_dcache_page(page);
                kunmap(page);

                host->head_pg.page = NULL;

                if (!force && sg_dma_len(sg) + sg->offset >
                    (host->page_idx << PAGE_SHIFT) + data->blksz - blk_head)
                        /* More blocks in this SG, don't unmap the next page */
                        return;
        }

        page = host->pg.page;
        if (!page)
                return;

        flush_dcache_page(page);
        kunmap(page);

        host->pg.page = NULL;
}

/* Called from MMC_WRITE_MULTIPLE_BLOCK or MMC_READ_MULTIPLE_BLOCK */
static void usdhi6_sg_advance(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;
        size_t done, total;

        /* New offset: set at the end of the previous block */
        if (host->head_pg.page) {
                /* Finished a cross-page block, jump to the new page */
                host->page_idx++;
                host->offset = data->blksz - host->head_len;
                host->blk_page = host->pg.mapped;
                usdhi6_sg_unmap(host, false);
        } else {
                host->offset += data->blksz;
                /* The completed block didn't cross a page boundary */
                if (host->offset == PAGE_SIZE) {
                        /* If required, we'll map the page below */
                        host->offset = 0;
                        host->page_idx++;
                }
        }

        /*
         * Now host->blk_page + host->offset point at the end of our last block
         * and host->page_idx is the index of the page, in which our new block
         * is located, if any
         */

        done = (host->page_idx << PAGE_SHIFT) + host->offset;
        total = host->sg->offset + sg_dma_len(host->sg);

        dev_dbg(mmc_dev(host->mmc), "%s(): %zu of %zu @ %zu\n", __func__,
                done, total, host->offset);

        if (done < total && host->offset) {
                /* More blocks in this page */
                if (host->offset + data->blksz > PAGE_SIZE)
                        /* We approached at a block, that spans 2 pages */
                        usdhi6_blk_bounce(host, host->sg);

                return;
        }

        /* Finished current page or an SG segment */
        usdhi6_sg_unmap(host, false);

        if (done == total) {
                /*
                 * End of an SG segment or the complete SG: jump to the next
                 * segment, we'll map it later in usdhi6_blk_read() or
                 * usdhi6_blk_write()
                 */
                struct scatterlist *next = sg_next(host->sg);

                host->page_idx = 0;

                if (!next)
                        host->wait = USDHI6_WAIT_FOR_DATA_END;
                host->sg = next;

                if (WARN(next && sg_dma_len(next) % data->blksz,
                         "SG size %u isn't a multiple of block size %u\n",
                         sg_dma_len(next), data->blksz))
                        data->error = -EINVAL;

                return;
        }

        /* We cannot get here after crossing a page border */

        /* Next page in the same SG */
        host->pg.page = nth_page(sg_page(host->sg), host->page_idx);
        host->pg.mapped = kmap(host->pg.page);
        host->blk_page = host->pg.mapped;

        dev_dbg(mmc_dev(host->mmc), "Mapped %p (%lx) at %p for CMD%u @ 0x%p\n",
                host->pg.page, page_to_pfn(host->pg.page), host->pg.mapped,
                host->mrq->cmd->opcode, host->mrq);
}

/*                      DMA handling                                    */

static void usdhi6_dma_release(struct usdhi6_host *host)
{
        host->dma_active = false;
        if (host->chan_tx) {
                struct dma_chan *chan = host->chan_tx;
                host->chan_tx = NULL;
                dma_release_channel(chan);
        }
        if (host->chan_rx) {
                struct dma_chan *chan = host->chan_rx;
                host->chan_rx = NULL;
                dma_release_channel(chan);
        }
}

static void usdhi6_dma_stop_unmap(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;

        if (!host->dma_active)
                return;

        usdhi6_write(host, USDHI6_CC_EXT_MODE, 0);
        host->dma_active = false;

        if (data->flags & MMC_DATA_READ)
                dma_unmap_sg(host->chan_rx->device->dev, data->sg,
                             data->sg_len, DMA_FROM_DEVICE);
        else
                dma_unmap_sg(host->chan_tx->device->dev, data->sg,
                             data->sg_len, DMA_TO_DEVICE);
}

static void usdhi6_dma_complete(void *arg)
{
        struct usdhi6_host *host = arg;
        struct mmc_request *mrq = host->mrq;

        if (WARN(!mrq || !mrq->data, "%s: NULL data in DMA completion for %p!\n",
                 dev_name(mmc_dev(host->mmc)), mrq))
                return;

        dev_dbg(mmc_dev(host->mmc), "%s(): CMD%u DMA completed\n", __func__,
                mrq->cmd->opcode);

        usdhi6_dma_stop_unmap(host);
        usdhi6_wait_for_brwe(host, mrq->data->flags & MMC_DATA_READ);
}

static int usdhi6_dma_setup(struct usdhi6_host *host, struct dma_chan *chan,
                            enum dma_transfer_direction dir)
{
        struct mmc_data *data = host->mrq->data;
        struct scatterlist *sg = data->sg;
        struct dma_async_tx_descriptor *desc = NULL;
        dma_cookie_t cookie = -EINVAL;
        enum dma_data_direction data_dir;
        int ret;

        switch (dir) {
        case DMA_MEM_TO_DEV:
                data_dir = DMA_TO_DEVICE;
                break;
        case DMA_DEV_TO_MEM:
                data_dir = DMA_FROM_DEVICE;
                break;
        default:
                return -EINVAL;
        }

        ret = dma_map_sg(chan->device->dev, sg, data->sg_len, data_dir);
        if (ret > 0) {
                host->dma_active = true;
                desc = dmaengine_prep_slave_sg(chan, sg, ret, dir,
                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
        }

        if (desc) {
                desc->callback = usdhi6_dma_complete;
                desc->callback_param = host;
                cookie = dmaengine_submit(desc);
        }

        dev_dbg(mmc_dev(host->mmc), "%s(): mapped %d -> %d, cookie %d @ %p\n",
                __func__, data->sg_len, ret, cookie, desc);

        if (cookie < 0) {
                /* DMA failed, fall back to PIO */
                if (ret >= 0)
                        ret = cookie;
                usdhi6_dma_release(host);
                dev_warn(mmc_dev(host->mmc),
                         "DMA failed: %d, falling back to PIO\n", ret);
        }

        return cookie;
}

static int usdhi6_dma_start(struct usdhi6_host *host)
{
        if (!host->chan_rx || !host->chan_tx)
                return -ENODEV;

        if (host->mrq->data->flags & MMC_DATA_READ)
                return usdhi6_dma_setup(host, host->chan_rx, DMA_DEV_TO_MEM);

        return usdhi6_dma_setup(host, host->chan_tx, DMA_MEM_TO_DEV);
}

static void usdhi6_dma_kill(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;

        dev_dbg(mmc_dev(host->mmc), "%s(): SG of %u: %ux%u\n",
                __func__, data->sg_len, data->blocks, data->blksz);
        /* Abort DMA */
        if (data->flags & MMC_DATA_READ)
                dmaengine_terminate_sync(host->chan_rx);
        else
                dmaengine_terminate_sync(host->chan_tx);
}

static void usdhi6_dma_check_error(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;

        dev_dbg(mmc_dev(host->mmc), "%s(): IO error %d, status 0x%x\n",
                __func__, host->io_error, usdhi6_read(host, USDHI6_SD_INFO1));

        if (host->io_error) {
                data->error = usdhi6_error_code(host);
                data->bytes_xfered = 0;
                usdhi6_dma_kill(host);
                usdhi6_dma_release(host);
                dev_warn(mmc_dev(host->mmc),
                         "DMA failed: %d, falling back to PIO\n", data->error);
                return;
        }

        /*
         * The datasheet tells us to check a response from the card, whereas
         * responses only come after the command phase, not after the data
         * phase. Let's check anyway.
         */
        if (host->irq_status & USDHI6_SD_INFO1_RSP_END)
                dev_warn(mmc_dev(host->mmc), "Unexpected response received!\n");
}

static void usdhi6_dma_kick(struct usdhi6_host *host)
{
        if (host->mrq->data->flags & MMC_DATA_READ)
                dma_async_issue_pending(host->chan_rx);
        else
                dma_async_issue_pending(host->chan_tx);
}

static void usdhi6_dma_request(struct usdhi6_host *host, phys_addr_t start)
{
        struct dma_slave_config cfg = {
                .src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
                .dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
        };
        int ret;

        host->chan_tx = dma_request_chan(mmc_dev(host->mmc), "tx");
        dev_dbg(mmc_dev(host->mmc), "%s: TX: got channel %p\n", __func__,
                host->chan_tx);

        if (IS_ERR(host->chan_tx)) {
                host->chan_tx = NULL;
                return;
        }

        cfg.direction = DMA_MEM_TO_DEV;
        cfg.dst_addr = start + USDHI6_SD_BUF0;
        cfg.dst_maxburst = 128; /* 128 words * 4 bytes = 512 bytes */
        cfg.src_addr = 0;
        ret = dmaengine_slave_config(host->chan_tx, &cfg);
        if (ret < 0)
                goto e_release_tx;

        host->chan_rx = dma_request_chan(mmc_dev(host->mmc), "rx");
        dev_dbg(mmc_dev(host->mmc), "%s: RX: got channel %p\n", __func__,
                host->chan_rx);

        if (IS_ERR(host->chan_rx)) {
                host->chan_rx = NULL;
                goto e_release_tx;
        }

        cfg.direction = DMA_DEV_TO_MEM;
        cfg.src_addr = cfg.dst_addr;
        cfg.src_maxburst = 128; /* 128 words * 4 bytes = 512 bytes */
        cfg.dst_addr = 0;
        ret = dmaengine_slave_config(host->chan_rx, &cfg);
        if (ret < 0)
                goto e_release_rx;

        return;

e_release_rx:
        dma_release_channel(host->chan_rx);
        host->chan_rx = NULL;
e_release_tx:
        dma_release_channel(host->chan_tx);
        host->chan_tx = NULL;
}

/*                      API helpers                                     */

static void usdhi6_clk_set(struct usdhi6_host *host, struct mmc_ios *ios)
{
        unsigned long rate = ios->clock;
        u32 val;
        unsigned int i;

        for (i = 1000; i; i--) {
                if (usdhi6_read(host, USDHI6_SD_INFO2) & USDHI6_SD_INFO2_SCLKDIVEN)
                        break;
                usleep_range(10, 100);
        }

        if (!i) {
                dev_err(mmc_dev(host->mmc), "SD bus busy, clock set aborted\n");
                return;
        }

        val = usdhi6_read(host, USDHI6_SD_CLK_CTRL) & ~USDHI6_SD_CLK_CTRL_DIV_MASK;

        if (rate) {
                unsigned long new_rate;

                if (host->imclk <= rate) {
                        if (ios->timing != MMC_TIMING_UHS_DDR50) {
                                /* Cannot have 1-to-1 clock in DDR mode */
                                new_rate = host->imclk;
                                val |= 0xff;
                        } else {
                                new_rate = host->imclk / 2;
                        }
                } else {
                        unsigned long div =
                                roundup_pow_of_two(DIV_ROUND_UP(host->imclk, rate));
                        val |= div >> 2;
                        new_rate = host->imclk / div;
                }

                if (host->rate == new_rate)
                        return;

                host->rate = new_rate;

                dev_dbg(mmc_dev(host->mmc), "target %lu, div %u, set %lu\n",
                        rate, (val & 0xff) << 2, new_rate);
        }

        /*
         * if old or new rate is equal to input rate, have to switch the clock
         * off before changing and on after
         */
        if (host->imclk == rate || host->imclk == host->rate || !rate)
                usdhi6_write(host, USDHI6_SD_CLK_CTRL,
                             val & ~USDHI6_SD_CLK_CTRL_SCLKEN);

        if (!rate) {
                host->rate = 0;
                return;
        }

        usdhi6_write(host, USDHI6_SD_CLK_CTRL, val);

        if (host->imclk == rate || host->imclk == host->rate ||
            !(val & USDHI6_SD_CLK_CTRL_SCLKEN))
                usdhi6_write(host, USDHI6_SD_CLK_CTRL,
                             val | USDHI6_SD_CLK_CTRL_SCLKEN);
}

static void usdhi6_set_power(struct usdhi6_host *host, struct mmc_ios *ios)
{
        struct mmc_host *mmc = host->mmc;

        if (!IS_ERR(mmc->supply.vmmc))
                /* Errors ignored... */
                mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,
                                      ios->power_mode ? ios->vdd : 0);
}

static int usdhi6_reset(struct usdhi6_host *host)
{
        int i;

        usdhi6_write(host, USDHI6_SOFT_RST, USDHI6_SOFT_RST_RESERVED);
        cpu_relax();
        usdhi6_write(host, USDHI6_SOFT_RST, USDHI6_SOFT_RST_RESERVED | USDHI6_SOFT_RST_RESET);
        for (i = 1000; i; i--)
                if (usdhi6_read(host, USDHI6_SOFT_RST) & USDHI6_SOFT_RST_RESET)
                        break;

        return i ? 0 : -ETIMEDOUT;
}

static void usdhi6_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
{
        struct usdhi6_host *host = mmc_priv(mmc);
        u32 option, mode;
        int ret;

        dev_dbg(mmc_dev(mmc), "%uHz, OCR: %u, power %u, bus-width %u, timing %u\n",
                ios->clock, ios->vdd, ios->power_mode, ios->bus_width, ios->timing);

        switch (ios->power_mode) {
        case MMC_POWER_OFF:
                usdhi6_set_power(host, ios);
                usdhi6_only_cd(host);
                break;
        case MMC_POWER_UP:
                /*
                 * We only also touch USDHI6_SD_OPTION from .request(), which
                 * cannot race with MMC_POWER_UP
                 */
                ret = usdhi6_reset(host);
                if (ret < 0) {
                        dev_err(mmc_dev(mmc), "Cannot reset the interface!\n");
                } else {
                        usdhi6_set_power(host, ios);
                        usdhi6_only_cd(host);
                }
                break;
        case MMC_POWER_ON:
                option = usdhi6_read(host, USDHI6_SD_OPTION);
                /*
                 * The eMMC standard only allows 4 or 8 bits in the DDR mode,
                 * the same probably holds for SD cards. We check here anyway,
                 * since the datasheet explicitly requires 4 bits for DDR.
                 */
                if (ios->bus_width == MMC_BUS_WIDTH_1) {
                        if (ios->timing == MMC_TIMING_UHS_DDR50)
                                dev_err(mmc_dev(mmc),
                                        "4 bits are required for DDR\n");
                        option |= USDHI6_SD_OPTION_WIDTH_1;
                        mode = 0;
                } else {
                        option &= ~USDHI6_SD_OPTION_WIDTH_1;
                        mode = ios->timing == MMC_TIMING_UHS_DDR50;
                }
                usdhi6_write(host, USDHI6_SD_OPTION, option);
                usdhi6_write(host, USDHI6_SDIF_MODE, mode);
                break;
        }

        if (host->rate != ios->clock)
                usdhi6_clk_set(host, ios);
}

/* This is data timeout. Response timeout is fixed to 640 clock cycles */
static void usdhi6_timeout_set(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        u32 val;
        unsigned long ticks;

        if (!mrq->data)
                ticks = host->rate / 1000 * mrq->cmd->busy_timeout;
        else
                ticks = host->rate / 1000000 * (mrq->data->timeout_ns / 1000) +
                        mrq->data->timeout_clks;

        if (!ticks || ticks > 1 << 27)
                /* Max timeout */
                val = 14;
        else if (ticks < 1 << 13)
                /* Min timeout */
                val = 0;
        else
                val = order_base_2(ticks) - 13;

        dev_dbg(mmc_dev(host->mmc), "Set %s timeout %lu ticks @ %lu Hz\n",
                mrq->data ? "data" : "cmd", ticks, host->rate);

        /* Timeout Counter mask: 0xf0 */
        usdhi6_write(host, USDHI6_SD_OPTION, (val << USDHI6_SD_OPTION_TIMEOUT_SHIFT) |
                     (usdhi6_read(host, USDHI6_SD_OPTION) & ~USDHI6_SD_OPTION_TIMEOUT_MASK));
}

static void usdhi6_request_done(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        struct mmc_data *data = mrq->data;

        if (WARN(host->pg.page || host->head_pg.page,
                 "Page %p or %p not unmapped: wait %u, CMD%d(%c) @ +0x%zx %ux%u in SG%u!\n",
                 host->pg.page, host->head_pg.page, host->wait, mrq->cmd->opcode,
                 data ? (data->flags & MMC_DATA_READ ? 'R' : 'W') : '-',
                 data ? host->offset : 0, data ? data->blocks : 0,
                 data ? data->blksz : 0, data ? data->sg_len : 0))
                usdhi6_sg_unmap(host, true);

        if (mrq->cmd->error ||
            (data && data->error) ||
            (mrq->stop && mrq->stop->error))
                dev_dbg(mmc_dev(host->mmc), "%s(CMD%d: %ux%u): err %d %d %d\n",
                        __func__, mrq->cmd->opcode, data ? data->blocks : 0,
                        data ? data->blksz : 0,
                        mrq->cmd->error,
                        data ? data->error : 1,
                        mrq->stop ? mrq->stop->error : 1);

        /* Disable DMA */
        usdhi6_write(host, USDHI6_CC_EXT_MODE, 0);
        host->wait = USDHI6_WAIT_FOR_REQUEST;
        host->mrq = NULL;

        mmc_request_done(host->mmc, mrq);
}

static int usdhi6_cmd_flags(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        struct mmc_command *cmd = mrq->cmd;
        u16 opc = cmd->opcode;

        if (host->app_cmd) {
                host->app_cmd = false;
                opc |= USDHI6_SD_CMD_APP;
        }

        if (mrq->data) {
                opc |= USDHI6_SD_CMD_DATA;

                if (mrq->data->flags & MMC_DATA_READ)
                        opc |= USDHI6_SD_CMD_READ;

                if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK ||
                    cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK ||
                    (cmd->opcode == SD_IO_RW_EXTENDED &&
                     mrq->data->blocks > 1)) {
                        opc |= USDHI6_SD_CMD_MULTI;
                        if (!mrq->stop)
                                opc |= USDHI6_SD_CMD_CMD12_AUTO_OFF;
                }

                switch (mmc_resp_type(cmd)) {
                case MMC_RSP_NONE:
                        opc |= USDHI6_SD_CMD_MODE_RSP_NONE;
                        break;
                case MMC_RSP_R1:
                        opc |= USDHI6_SD_CMD_MODE_RSP_R1;
                        break;
                case MMC_RSP_R1B:
                        opc |= USDHI6_SD_CMD_MODE_RSP_R1B;
                        break;
                case MMC_RSP_R2:
                        opc |= USDHI6_SD_CMD_MODE_RSP_R2;
                        break;
                case MMC_RSP_R3:
                        opc |= USDHI6_SD_CMD_MODE_RSP_R3;
                        break;
                default:
                        dev_warn(mmc_dev(host->mmc),
                                 "Unknown response type %d\n",
                                 mmc_resp_type(cmd));
                        return -EINVAL;
                }
        }

        return opc;
}

static int usdhi6_rq_start(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        struct mmc_command *cmd = mrq->cmd;
        struct mmc_data *data = mrq->data;
        int opc = usdhi6_cmd_flags(host);
        int i;

        if (opc < 0)
                return opc;

        for (i = 1000; i; i--) {
                if (!(usdhi6_read(host, USDHI6_SD_INFO2) & USDHI6_SD_INFO2_CBSY))
                        break;
                usleep_range(10, 100);
        }

        if (!i) {
                dev_dbg(mmc_dev(host->mmc), "Command active, request aborted\n");
                return -EAGAIN;
        }

        if (data) {
                bool use_dma;
                int ret = 0;

                host->page_idx = 0;

                if (cmd->opcode == SD_IO_RW_EXTENDED && data->blocks > 1) {
                        switch (data->blksz) {
                        case 512:
                                break;
                        case 32:
                        case 64:
                        case 128:
                        case 256:
                                if (mrq->stop)
                                        ret = -EINVAL;
                                break;
                        default:
                                ret = -EINVAL;
                        }
                } else if ((cmd->opcode == MMC_READ_MULTIPLE_BLOCK ||
                            cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK) &&
                           data->blksz != 512) {
                        ret = -EINVAL;
                }

                if (ret < 0) {
                        dev_warn(mmc_dev(host->mmc), "%s(): %u blocks of %u bytes\n",
                                 __func__, data->blocks, data->blksz);
                        return -EINVAL;
                }

                if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK ||
                    cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK ||
                    (cmd->opcode == SD_IO_RW_EXTENDED &&
                     data->blocks > 1))
                        usdhi6_sg_prep(host);

                usdhi6_write(host, USDHI6_SD_SIZE, data->blksz);

                if ((data->blksz >= USDHI6_MIN_DMA ||
                     data->blocks > 1) &&
                    (data->blksz % 4 ||
                     data->sg->offset % 4))
                        dev_dbg(mmc_dev(host->mmc),
                                "Bad SG of %u: %ux%u @ %u\n", data->sg_len,
                                data->blksz, data->blocks, data->sg->offset);

                /* Enable DMA for USDHI6_MIN_DMA bytes or more */
                use_dma = data->blksz >= USDHI6_MIN_DMA &&
                        !(data->blksz % 4) &&
                        usdhi6_dma_start(host) >= DMA_MIN_COOKIE;

                if (use_dma)
                        usdhi6_write(host, USDHI6_CC_EXT_MODE, USDHI6_CC_EXT_MODE_SDRW);

                dev_dbg(mmc_dev(host->mmc),
                        "%s(): request opcode %u, %u blocks of %u bytes in %u segments, %s %s @+0x%x%s\n",
                        __func__, cmd->opcode, data->blocks, data->blksz,
                        data->sg_len, use_dma ? "DMA" : "PIO",
                        data->flags & MMC_DATA_READ ? "read" : "write",
                        data->sg->offset, mrq->stop ? " + stop" : "");
        } else {
                dev_dbg(mmc_dev(host->mmc), "%s(): request opcode %u\n",
                        __func__, cmd->opcode);
        }

        /* We have to get a command completion interrupt with DMA too */
        usdhi6_wait_for_resp(host);

        host->wait = USDHI6_WAIT_FOR_CMD;
        schedule_delayed_work(&host->timeout_work, host->timeout);

        /* SEC bit is required to enable block counting by the core */
        usdhi6_write(host, USDHI6_SD_STOP,
                     data && data->blocks > 1 ? USDHI6_SD_STOP_SEC : 0);
        usdhi6_write(host, USDHI6_SD_ARG, cmd->arg);

        /* Kick command execution */
        usdhi6_write(host, USDHI6_SD_CMD, opc);

        return 0;
}

static void usdhi6_request(struct mmc_host *mmc, struct mmc_request *mrq)
{
        struct usdhi6_host *host = mmc_priv(mmc);
        int ret;

        cancel_delayed_work_sync(&host->timeout_work);

        host->mrq = mrq;
        host->sg = NULL;

        usdhi6_timeout_set(host);
        ret = usdhi6_rq_start(host);
        if (ret < 0) {
                mrq->cmd->error = ret;
                usdhi6_request_done(host);
        }
}

static int usdhi6_get_cd(struct mmc_host *mmc)
{
        struct usdhi6_host *host = mmc_priv(mmc);
        /* Read is atomic, no need to lock */
        u32 status = usdhi6_read(host, USDHI6_SD_INFO1) & USDHI6_SD_INFO1_CD;

/*
 *      level   status.CD       CD_ACTIVE_HIGH  card present
 *      1       0               0               0
 *      1       0               1               1
 *      0       1               0               1
 *      0       1               1               0
 */
        return !status ^ !(mmc->caps2 & MMC_CAP2_CD_ACTIVE_HIGH);
}

static int usdhi6_get_ro(struct mmc_host *mmc)
{
        struct usdhi6_host *host = mmc_priv(mmc);
        /* No locking as above */
        u32 status = usdhi6_read(host, USDHI6_SD_INFO1) & USDHI6_SD_INFO1_WP;

/*
 *      level   status.WP       RO_ACTIVE_HIGH  card read-only
 *      1       0               0               0
 *      1       0               1               1
 *      0       1               0               1
 *      0       1               1               0
 */
        return !status ^ !(mmc->caps2 & MMC_CAP2_RO_ACTIVE_HIGH);
}

static void usdhi6_enable_sdio_irq(struct mmc_host *mmc, int enable)
{
        struct usdhi6_host *host = mmc_priv(mmc);

        dev_dbg(mmc_dev(mmc), "%s(): %sable\n", __func__, enable ? "en" : "dis");

        if (enable) {
                host->sdio_mask = USDHI6_SDIO_INFO1_IRQ & ~USDHI6_SDIO_INFO1_IOIRQ;
                usdhi6_write(host, USDHI6_SDIO_INFO1_MASK, host->sdio_mask);
                usdhi6_write(host, USDHI6_SDIO_MODE, 1);
        } else {
                usdhi6_write(host, USDHI6_SDIO_MODE, 0);
                usdhi6_write(host, USDHI6_SDIO_INFO1_MASK, USDHI6_SDIO_INFO1_IRQ);
                host->sdio_mask = USDHI6_SDIO_INFO1_IRQ;
        }
}

static int usdhi6_set_pinstates(struct usdhi6_host *host, int voltage)
{
        if (IS_ERR(host->pins_uhs))
                return 0;

        switch (voltage) {
        case MMC_SIGNAL_VOLTAGE_180:
        case MMC_SIGNAL_VOLTAGE_120:
                return pinctrl_select_state(host->pinctrl,
                                            host->pins_uhs);

        default:
                return pinctrl_select_default_state(mmc_dev(host->mmc));
        }
}

static int usdhi6_sig_volt_switch(struct mmc_host *mmc, struct mmc_ios *ios)
{
        int ret;

        ret = mmc_regulator_set_vqmmc(mmc, ios);
        if (ret < 0)
                return ret;

        ret = usdhi6_set_pinstates(mmc_priv(mmc), ios->signal_voltage);
        if (ret)
                dev_warn_once(mmc_dev(mmc),
                              "Failed to set pinstate err=%d\n", ret);
        return ret;
}

static int usdhi6_card_busy(struct mmc_host *mmc)
{
        struct usdhi6_host *host = mmc_priv(mmc);
        u32 tmp = usdhi6_read(host, USDHI6_SD_INFO2);

        /* Card is busy if it is pulling dat[0] low */
        return !(tmp & USDHI6_SD_INFO2_SDDAT0);
}

static const struct mmc_host_ops usdhi6_ops = {
        .request        = usdhi6_request,
        .set_ios        = usdhi6_set_ios,
        .get_cd         = usdhi6_get_cd,
        .get_ro         = usdhi6_get_ro,
        .enable_sdio_irq = usdhi6_enable_sdio_irq,
        .start_signal_voltage_switch = usdhi6_sig_volt_switch,
        .card_busy = usdhi6_card_busy,
};

/*                      State machine handlers                          */

static void usdhi6_resp_cmd12(struct usdhi6_host *host)
{
        struct mmc_command *cmd = host->mrq->stop;
        cmd->resp[0] = usdhi6_read(host, USDHI6_SD_RSP10);
}

static void usdhi6_resp_read(struct usdhi6_host *host)
{
        struct mmc_command *cmd = host->mrq->cmd;
        u32 *rsp = cmd->resp, tmp = 0;
        int i;

/*
 * RSP10        39-8
 * RSP32        71-40
 * RSP54        103-72
 * RSP76        127-104
 * R2-type response:
 * resp[0]      = r[127..96]
 * resp[1]      = r[95..64]
 * resp[2]      = r[63..32]
 * resp[3]      = r[31..0]
 * Other responses:
 * resp[0]      = r[39..8]
 */

        if (mmc_resp_type(cmd) == MMC_RSP_NONE)
                return;

        if (!(host->irq_status & USDHI6_SD_INFO1_RSP_END)) {
                dev_err(mmc_dev(host->mmc),
                        "CMD%d: response expected but is missing!\n", cmd->opcode);
                return;
        }

        if (mmc_resp_type(cmd) & MMC_RSP_136)
                for (i = 0; i < 4; i++) {
                        if (i)
                                rsp[3 - i] = tmp >> 24;
                        tmp = usdhi6_read(host, USDHI6_SD_RSP10 + i * 8);
                        rsp[3 - i] |= tmp << 8;
                }
        else if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK ||
                 cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK)
                /* Read RSP54 to avoid conflict with auto CMD12 */
                rsp[0] = usdhi6_read(host, USDHI6_SD_RSP54);
        else
                rsp[0] = usdhi6_read(host, USDHI6_SD_RSP10);

        dev_dbg(mmc_dev(host->mmc), "Response 0x%x\n", rsp[0]);
}

static int usdhi6_blk_read(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;
        u32 *p;
        int i, rest;

        if (host->io_error) {
                data->error = usdhi6_error_code(host);
                goto error;
        }

        if (host->pg.page) {
                p = host->blk_page + host->offset;
        } else {
                p = usdhi6_sg_map(host);
                if (!p) {
                        data->error = -ENOMEM;
                        goto error;
                }
        }

        for (i = 0; i < data->blksz / 4; i++, p++)
                *p = usdhi6_read(host, USDHI6_SD_BUF0);

        rest = data->blksz % 4;
        for (i = 0; i < (rest + 1) / 2; i++) {
                u16 d = usdhi6_read16(host, USDHI6_SD_BUF0);
                ((u8 *)p)[2 * i] = ((u8 *)&d)[0];
                if (rest > 1 && !i)
                        ((u8 *)p)[2 * i + 1] = ((u8 *)&d)[1];
        }

        return 0;

error:
        dev_dbg(mmc_dev(host->mmc), "%s(): %d\n", __func__, data->error);
        host->wait = USDHI6_WAIT_FOR_REQUEST;
        return data->error;
}

static int usdhi6_blk_write(struct usdhi6_host *host)
{
        struct mmc_data *data = host->mrq->data;
        u32 *p;
        int i, rest;

        if (host->io_error) {
                data->error = usdhi6_error_code(host);
                goto error;
        }

        if (host->pg.page) {
                p = host->blk_page + host->offset;
        } else {
                p = usdhi6_sg_map(host);
                if (!p) {
                        data->error = -ENOMEM;
                        goto error;
                }
        }

        for (i = 0; i < data->blksz / 4; i++, p++)
                usdhi6_write(host, USDHI6_SD_BUF0, *p);

        rest = data->blksz % 4;
        for (i = 0; i < (rest + 1) / 2; i++) {
                u16 d;
                ((u8 *)&d)[0] = ((u8 *)p)[2 * i];
                if (rest > 1 && !i)
                        ((u8 *)&d)[1] = ((u8 *)p)[2 * i + 1];
                else
                        ((u8 *)&d)[1] = 0;
                usdhi6_write16(host, USDHI6_SD_BUF0, d);
        }

        return 0;

error:
        dev_dbg(mmc_dev(host->mmc), "%s(): %d\n", __func__, data->error);
        host->wait = USDHI6_WAIT_FOR_REQUEST;
        return data->error;
}

static int usdhi6_stop_cmd(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;

        switch (mrq->cmd->opcode) {
        case MMC_READ_MULTIPLE_BLOCK:
        case MMC_WRITE_MULTIPLE_BLOCK:
                if (mrq->stop->opcode == MMC_STOP_TRANSMISSION) {
                        host->wait = USDHI6_WAIT_FOR_STOP;
                        return 0;
                }
                fallthrough;    /* Unsupported STOP command */
        default:
                dev_err(mmc_dev(host->mmc),
                        "unsupported stop CMD%d for CMD%d\n",
                        mrq->stop->opcode, mrq->cmd->opcode);
                mrq->stop->error = -EOPNOTSUPP;
        }

        return -EOPNOTSUPP;
}

static bool usdhi6_end_cmd(struct usdhi6_host *host)
{
        struct mmc_request *mrq = host->mrq;
        struct mmc_command *cmd = mrq->cmd;

        if (host->io_error) {
                cmd->error = usdhi6_error_code(host);
                return false;
        }

        usdhi6_resp_read(host);

        if (!mrq->data)
                return false;

        if (host->dma_active) {
                usdhi6_dma_kick(host);
                if (!mrq->stop)
                        host->wait = USDHI6_WAIT_FOR_DMA;
                else if (usdhi6_stop_cmd(host) < 0)
                        return false;
        } else if (mrq->data->flags & MMC_DATA_READ) {
                if (cmd->opcode == MMC_READ_MULTIPLE_BLOCK ||
                    (cmd->opcode == SD_IO_RW_EXTENDED &&
                     mrq->data->blocks > 1))
                        host->wait = USDHI6_WAIT_FOR_MREAD;
                else
                        host->wait = USDHI6_WAIT_FOR_READ;
        } else {
                if (cmd->opcode == MMC_WRITE_MULTIPLE_BLOCK ||
                    (cmd->opcode == SD_IO_RW_EXTENDED &&
                     mrq->data->blocks > 1))
                        host->wait = USDHI6_WAIT_FOR_MWRITE;
                else
                        host->wait = USDHI6_WAIT_FOR_WRITE;
        }

        return true;
}

static bool usdhi6_read_block(struct usdhi6_host *host)
{
        /* ACCESS_END IRQ is already unmasked */
        int ret = usdhi6_blk_read(host);

        /*
         * Have to force unmapping both pages: the single block could have been
         * cross-page, in which case for single-block IO host->page_idx == 0.
         * So, if we don't force, the second page won't be unmapped.
         */
        usdhi6_sg_unmap(host, true);

        if (ret < 0)
                return false;

        host->wait = USDHI6_WAIT_FOR_DATA_END;
        return true;
}

static bool usdhi6_mread_block(struct usdhi6_host *host)
{
        int ret = usdhi6_blk_read(host);

        if (ret < 0)
                return false;

        usdhi6_sg_advance(host);

        return !host->mrq->data->error &&
                (host->wait != USDHI6_WAIT_FOR_DATA_END || !host->mrq->stop);
}

static bool usdhi6_write_block(struct usdhi6_host *host)
{
        int ret = usdhi6_blk_write(host);

        /* See comment in usdhi6_read_block() */
        usdhi6_sg_unmap(host, true);

        if (ret < 0)
                return false;

        host->wait = USDHI6_WAIT_FOR_DATA_END;
        return true;
}

static bool usdhi6_mwrite_block(struct usdhi6_host *host)
{
        int ret = usdhi6_blk_write(host);

        if (ret < 0)
                return false;

        usdhi6_sg_advance(host);

        return !host->mrq->data->error &&
                (host->wait != USDHI6_WAIT_FOR_DATA_END || !host->mrq->stop);
}

/*                      Interrupt & timeout handlers                    */

static irqreturn_t usdhi6_sd_bh(int irq, void *dev_id)
{
        struct usdhi6_host *host = dev_id;
        struct mmc_request *mrq;
        struct mmc_command *cmd;
        struct mmc_data *data;
        bool io_wait = false;

        cancel_delayed_work_sync(&host->timeout_work);

        mrq = host->mrq;
        if (!mrq)
                return IRQ_HANDLED;

        cmd = mrq->cmd;
        data = mrq->data;

        switch (host->wait) {
        case USDHI6_WAIT_FOR_REQUEST:
                /* We're too late, the timeout has already kicked in */
                return IRQ_HANDLED;
        case USDHI6_WAIT_FOR_CMD:
                /* Wait for data? */
                io_wait = usdhi6_end_cmd(host);
                break;
        case USDHI6_WAIT_FOR_MREAD:
                /* Wait for more data? */
                io_wait = usdhi6_mread_block(host);
                break;
        case USDHI6_WAIT_FOR_READ:
                /* Wait for data end? */
                io_wait = usdhi6_read_block(host);
                break;
        case USDHI6_WAIT_FOR_MWRITE:
                /* Wait data to write? */
                io_wait = usdhi6_mwrite_block(host);
                break;
        case USDHI6_WAIT_FOR_WRITE:
                /* Wait for data end? */
                io_wait = usdhi6_write_block(host);
                break;
        case USDHI6_WAIT_FOR_DMA:
                usdhi6_dma_check_error(host);
                break;
        case USDHI6_WAIT_FOR_STOP:
                usdhi6_write(host, USDHI6_SD_STOP, 0);
                if (host->io_error) {
                        int ret = usdhi6_error_code(host);
                        if (mrq->stop)
                                mrq->stop->error = ret;
                        else
                                mrq->data->error = ret;
                        dev_warn(mmc_dev(host->mmc), "%s(): %d\n", __func__, ret);
                        break;
                }
                usdhi6_resp_cmd12(host);
                mrq->stop->error = 0;
                break;
        case USDHI6_WAIT_FOR_DATA_END:
                if (host->io_error) {
                        mrq->data->error = usdhi6_error_code(host);
                        dev_warn(mmc_dev(host->mmc), "%s(): %d\n", __func__,
                                 mrq->data->error);
                }
                break;
        default:
                cmd->error = -EFAULT;
                dev_err(mmc_dev(host->mmc), "Invalid state %u\n", host->wait);
                usdhi6_request_done(host);
                return IRQ_HANDLED;
        }

        if (io_wait) {
                schedule_delayed_work(&host->timeout_work, host->timeout);
                /* Wait for more data or ACCESS_END */
                if (!host->dma_active)
                        usdhi6_wait_for_brwe(host, mrq->data->flags & MMC_DATA_READ);
                return IRQ_HANDLED;
        }

        if (!cmd->error) {
                if (data) {
                        if (!data->error) {
                                if (host->wait != USDHI6_WAIT_FOR_STOP &&
                                    host->mrq->stop &&
                                    !host->mrq->stop->error &&
                                    !usdhi6_stop_cmd(host)) {
                                        /* Sending STOP */
                                        usdhi6_wait_for_resp(host);

                                        schedule_delayed_work(&host->timeout_work,
                                                              host->timeout);

                                        return IRQ_HANDLED;
                                }

                                data->bytes_xfered = data->blocks * data->blksz;
                        } else {
                                /* Data error: might need to unmap the last page */
                                dev_warn(mmc_dev(host->mmc), "%s(): data error %d\n",
                                         __func__, data->error);
                                usdhi6_sg_unmap(host, true);
                        }
                } else if (cmd->opcode == MMC_APP_CMD) {
                        host->app_cmd = true;
                }
        }

        usdhi6_request_done(host);

        return IRQ_HANDLED;
}

static irqreturn_t usdhi6_sd(int irq, void *dev_id)
{
        struct usdhi6_host *host = dev_id;
        u16 status, status2, error;

        status = usdhi6_read(host, USDHI6_SD_INFO1) & ~host->status_mask &
                ~USDHI6_SD_INFO1_CARD;
        status2 = usdhi6_read(host, USDHI6_SD_INFO2) & ~host->status2_mask;

        usdhi6_only_cd(host);

        dev_dbg(mmc_dev(host->mmc),
                "IRQ status = 0x%08x, status2 = 0x%08x\n", status, status2);

        if (!status && !status2)
                return IRQ_NONE;

        error = status2 & USDHI6_SD_INFO2_ERR;

        /* Ack / clear interrupts */
        if (USDHI6_SD_INFO1_IRQ & status)
                usdhi6_write(host, USDHI6_SD_INFO1,
                             0xffff & ~(USDHI6_SD_INFO1_IRQ & status));

        if (USDHI6_SD_INFO2_IRQ & status2) {
                if (error)
                        /* In error cases BWE and BRE aren't cleared automatically */
                        status2 |= USDHI6_SD_INFO2_BWE | USDHI6_SD_INFO2_BRE;

                usdhi6_write(host, USDHI6_SD_INFO2,
                             0xffff & ~(USDHI6_SD_INFO2_IRQ & status2));
        }

        host->io_error = error;
        host->irq_status = status;

        if (error) {
                /* Don't pollute the log with unsupported command timeouts */
                if (host->wait != USDHI6_WAIT_FOR_CMD ||
                    error != USDHI6_SD_INFO2_RSP_TOUT)
                        dev_warn(mmc_dev(host->mmc),
                                 "%s(): INFO2 error bits 0x%08x\n",
                                 __func__, error);
                else
                        dev_dbg(mmc_dev(host->mmc),
                                "%s(): INFO2 error bits 0x%08x\n",
                                __func__, error);
        }

        return IRQ_WAKE_THREAD;
}

static irqreturn_t usdhi6_sdio(int irq, void *dev_id)
{
        struct usdhi6_host *host = dev_id;
        u32 status = usdhi6_read(host, USDHI6_SDIO_INFO1) & ~host->sdio_mask;

        dev_dbg(mmc_dev(host->mmc), "%s(): status 0x%x\n", __func__, status);

        if (!status)
                return IRQ_NONE;

        usdhi6_write(host, USDHI6_SDIO_INFO1, ~status);

        mmc_signal_sdio_irq(host->mmc);

        return IRQ_HANDLED;
}

static irqreturn_t usdhi6_cd(int irq, void *dev_id)
{
        struct usdhi6_host *host = dev_id;
        struct mmc_host *mmc = host->mmc;
        u16 status;

        /* We're only interested in hotplug events here */
        status = usdhi6_read(host, USDHI6_SD_INFO1) & ~host->status_mask &
                USDHI6_SD_INFO1_CARD;

        if (!status)
                return IRQ_NONE;

        /* Ack */
        usdhi6_write(host, USDHI6_SD_INFO1, ~status);

        if (!work_pending(&mmc->detect.work) &&
            (((status & USDHI6_SD_INFO1_CARD_INSERT) &&
              !mmc->card) ||
             ((status & USDHI6_SD_INFO1_CARD_EJECT) &&
              mmc->card)))
                mmc_detect_change(mmc, msecs_to_jiffies(100));

        return IRQ_HANDLED;
}

/*
 * Actually this should not be needed, if the built-in timeout works reliably in
 * the both PIO cases and DMA never fails. But if DMA does fail, a timeout
 * handler might be the only way to catch the error.
 */
static void usdhi6_timeout_work(struct work_struct *work)
{
        struct delayed_work *d = to_delayed_work(work);
        struct usdhi6_host *host = container_of(d, struct usdhi6_host, timeout_work);
        struct mmc_request *mrq = host->mrq;
        struct mmc_data *data = mrq ? mrq->data : NULL;
        struct scatterlist *sg;

        dev_warn(mmc_dev(host->mmc),
                 "%s timeout wait %u CMD%d: IRQ 0x%08x:0x%08x, last IRQ 0x%08x\n",
                 host->dma_active ? "DMA" : "PIO",
                 host->wait, mrq ? mrq->cmd->opcode : -1,
                 usdhi6_read(host, USDHI6_SD_INFO1),
                 usdhi6_read(host, USDHI6_SD_INFO2), host->irq_status);

        if (host->dma_active) {
                usdhi6_dma_kill(host);
                usdhi6_dma_stop_unmap(host);
        }

        switch (host->wait) {
        default:
                dev_err(mmc_dev(host->mmc), "Invalid state %u\n", host->wait);
                fallthrough;    /* mrq can be NULL, but is impossible */
        case USDHI6_WAIT_FOR_CMD:
                usdhi6_error_code(host);
                if (mrq)
                        mrq->cmd->error = -ETIMEDOUT;
                break;
        case USDHI6_WAIT_FOR_STOP:
                usdhi6_error_code(host);
                mrq->stop->error = -ETIMEDOUT;
                break;
        case USDHI6_WAIT_FOR_DMA:
        case USDHI6_WAIT_FOR_MREAD:
        case USDHI6_WAIT_FOR_MWRITE:
        case USDHI6_WAIT_FOR_READ:
        case USDHI6_WAIT_FOR_WRITE:
                sg = host->sg ?: data->sg;
                dev_dbg(mmc_dev(host->mmc),
                        "%c: page #%u @ +0x%zx %ux%u in SG%u. Current SG %u bytes @ %u\n",
                        data->flags & MMC_DATA_READ ? 'R' : 'W', host->page_idx,
                        host->offset, data->blocks, data->blksz, data->sg_len,
                        sg_dma_len(sg), sg->offset);
                usdhi6_sg_unmap(host, true);
                fallthrough;    /* page unmapped in USDHI6_WAIT_FOR_DATA_END */
        case USDHI6_WAIT_FOR_DATA_END:
                usdhi6_error_code(host);
                data->error = -ETIMEDOUT;
        }

        if (mrq)
                usdhi6_request_done(host);
}

/*                       Probe / release                                */

static const struct of_device_id usdhi6_of_match[] = {
        {.compatible = "renesas,usdhi6rol0"},
        {}
};
MODULE_DEVICE_TABLE(of, usdhi6_of_match);

static int usdhi6_probe(struct platform_device *pdev)
{
        struct device *dev = &pdev->dev;
        struct mmc_host *mmc;
        struct usdhi6_host *host;
        struct resource *res;
        int irq_cd, irq_sd, irq_sdio;
        u32 version;
        int ret;

        if (!dev->of_node)
                return -ENODEV;

        irq_cd = platform_get_irq_byname(pdev, "card detect");
        irq_sd = platform_get_irq_byname(pdev, "data");
        irq_sdio = platform_get_irq_byname(pdev, "SDIO");
        if (irq_sd < 0 || irq_sdio < 0)
                return -ENODEV;

        mmc = mmc_alloc_host(sizeof(struct usdhi6_host), dev);
        if (!mmc)
                return -ENOMEM;

        ret = mmc_regulator_get_supply(mmc);
        if (ret)
                goto e_free_mmc;

        ret = mmc_of_parse(mmc);
        if (ret < 0)
                goto e_free_mmc;

        host            = mmc_priv(mmc);
        host->mmc       = mmc;
        host->wait      = USDHI6_WAIT_FOR_REQUEST;
        host->timeout   = msecs_to_jiffies(USDHI6_REQ_TIMEOUT_MS);
        /*
         * We use a fixed timeout of 4s, hence inform the core about it. A
         * future improvement should instead respect the cmd->busy_timeout.
         */
        mmc->max_busy_timeout = USDHI6_REQ_TIMEOUT_MS;

        host->pinctrl = devm_pinctrl_get(&pdev->dev);
        if (IS_ERR(host->pinctrl)) {
                ret = PTR_ERR(host->pinctrl);
                goto e_free_mmc;
        }

        host->pins_uhs = pinctrl_lookup_state(host->pinctrl, "state_uhs");

        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
        host->base = devm_ioremap_resource(dev, res);
        if (IS_ERR(host->base)) {
                ret = PTR_ERR(host->base);
                goto e_free_mmc;
        }

        host->clk = devm_clk_get(dev, NULL);
        if (IS_ERR(host->clk)) {
                ret = PTR_ERR(host->clk);
                goto e_free_mmc;
        }

        host->imclk = clk_get_rate(host->clk);

        ret = clk_prepare_enable(host->clk);
        if (ret < 0)
                goto e_free_mmc;

        version = usdhi6_read(host, USDHI6_VERSION);
        if ((version & 0xfff) != 0xa0d) {
                ret = -EPERM;
                dev_err(dev, "Version not recognized %x\n", version);
                goto e_clk_off;
        }

        dev_info(dev, "A USDHI6ROL0 SD host detected with %d ports\n",
                 usdhi6_read(host, USDHI6_SD_PORT_SEL) >> USDHI6_SD_PORT_SEL_PORTS_SHIFT);

        usdhi6_mask_all(host);

        if (irq_cd >= 0) {
                ret = devm_request_irq(dev, irq_cd, usdhi6_cd, 0,
                                       dev_name(dev), host);
                if (ret < 0)
                        goto e_clk_off;
        } else {
                mmc->caps |= MMC_CAP_NEEDS_POLL;
        }

        ret = devm_request_threaded_irq(dev, irq_sd, usdhi6_sd, usdhi6_sd_bh, 0,
                               dev_name(dev), host);
        if (ret < 0)
                goto e_clk_off;

        ret = devm_request_irq(dev, irq_sdio, usdhi6_sdio, 0,
                               dev_name(dev), host);
        if (ret < 0)
                goto e_clk_off;

        INIT_DELAYED_WORK(&host->timeout_work, usdhi6_timeout_work);

        usdhi6_dma_request(host, res->start);

        mmc->ops = &usdhi6_ops;
        mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
                     MMC_CAP_SDIO_IRQ;
        /* Set .max_segs to some random number. Feel free to adjust. */
        mmc->max_segs = 32;
        mmc->max_blk_size = 512;
        mmc->max_req_size = PAGE_SIZE * mmc->max_segs;
        mmc->max_blk_count = mmc->max_req_size / mmc->max_blk_size;
        /*
         * Setting .max_seg_size to 1 page would simplify our page-mapping code,
         * But OTOH, having large segments makes DMA more efficient. We could
         * check, whether we managed to get DMA and fall back to 1 page
         * segments, but if we do manage to obtain DMA and then it fails at
         * run-time and we fall back to PIO, we will continue getting large
         * segments. So, we wouldn't be able to get rid of the code anyway.
         */
        mmc->max_seg_size = mmc->max_req_size;
        if (!mmc->f_max)
                mmc->f_max = host->imclk;
        mmc->f_min = host->imclk / 512;

        platform_set_drvdata(pdev, host);

        ret = mmc_add_host(mmc);
        if (ret < 0)
                goto e_release_dma;

        return 0;

e_release_dma:
        usdhi6_dma_release(host);
e_clk_off:
        clk_disable_unprepare(host->clk);
e_free_mmc:
        mmc_free_host(mmc);

        return ret;
}

static int usdhi6_remove(struct platform_device *pdev)
{
        struct usdhi6_host *host = platform_get_drvdata(pdev);

        mmc_remove_host(host->mmc);

        usdhi6_mask_all(host);
        cancel_delayed_work_sync(&host->timeout_work);
        usdhi6_dma_release(host);
        clk_disable_unprepare(host->clk);
        mmc_free_host(host->mmc);

        return 0;
}

static struct platform_driver usdhi6_driver = {
        .probe          = usdhi6_probe,
        .remove         = usdhi6_remove,
        .driver         = {
                .name   = "usdhi6rol0",
                .probe_type = PROBE_PREFER_ASYNCHRONOUS,
                .of_match_table = usdhi6_of_match,
        },
};

module_platform_driver(usdhi6_driver);

MODULE_DESCRIPTION("Renesas usdhi6rol0 SD/SDIO host driver");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS("platform:usdhi6rol0");
MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");