LXR linux/drivers/dma/ptdma/ptdma-dev.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * AMD Passthru DMA device driver
   4 * -- Based on the CCP driver
   5 *
   6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
   7 *
   8 * Author: Sanjay R Mehta <sanju.mehta@amd.com>
   9 * Author: Gary R Hook <gary.hook@amd.com>
  10 */
  11
  12#include <linux/bitfield.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/debugfs.h>
  15#include <linux/interrupt.h>
  16#include <linux/kernel.h>
  17#include <linux/module.h>
  18#include <linux/pci.h>
  19
  20#include "ptdma.h"
  21
  22/* Human-readable error strings */
  23static char *pt_error_codes[] = {
  24        "",
  25        "ERR 01: ILLEGAL_ENGINE",
  26        "ERR 03: ILLEGAL_FUNCTION_TYPE",
  27        "ERR 04: ILLEGAL_FUNCTION_MODE",
  28        "ERR 06: ILLEGAL_FUNCTION_SIZE",
  29        "ERR 08: ILLEGAL_FUNCTION_RSVD",
  30        "ERR 09: ILLEGAL_BUFFER_LENGTH",
  31        "ERR 10: VLSB_FAULT",
  32        "ERR 11: ILLEGAL_MEM_ADDR",
  33        "ERR 12: ILLEGAL_MEM_SEL",
  34        "ERR 13: ILLEGAL_CONTEXT_ID",
  35        "ERR 15: 0xF Reserved",
  36        "ERR 18: CMD_TIMEOUT",
  37        "ERR 19: IDMA0_AXI_SLVERR",
  38        "ERR 20: IDMA0_AXI_DECERR",
  39        "ERR 21: 0x15 Reserved",
  40        "ERR 22: IDMA1_AXI_SLAVE_FAULT",
  41        "ERR 23: IDMA1_AIXI_DECERR",
  42        "ERR 24: 0x18 Reserved",
  43        "ERR 27: 0x1B Reserved",
  44        "ERR 38: ODMA0_AXI_SLVERR",
  45        "ERR 39: ODMA0_AXI_DECERR",
  46        "ERR 40: 0x28 Reserved",
  47        "ERR 41: ODMA1_AXI_SLVERR",
  48        "ERR 42: ODMA1_AXI_DECERR",
  49        "ERR 43: LSB_PARITY_ERR",
  50};
  51
  52static void pt_log_error(struct pt_device *d, int e)
  53{
  54        dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
  55}
  56
  57void pt_start_queue(struct pt_cmd_queue *cmd_q)
  58{
  59        /* Turn on the run bit */
  60        iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
  61}
  62
  63void pt_stop_queue(struct pt_cmd_queue *cmd_q)
  64{
  65        /* Turn off the run bit */
  66        iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
  67}
  68
  69static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
  70{
  71        bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
  72        u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
  73        u32 tail;
  74
  75        if (soc) {
  76                desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
  77                desc->dw0 &= ~DWORD0_SOC;
  78        }
  79        mutex_lock(&cmd_q->q_mutex);
  80
  81        /* Copy 32-byte command descriptor to hw queue. */
  82        memcpy(q_desc, desc, 32);
  83        cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
  84
  85        /* The data used by this command must be flushed to memory */
  86        wmb();
  87
  88        /* Write the new tail address back to the queue register */
  89        tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
  90        iowrite32(tail, cmd_q->reg_control + 0x0004);
  91
  92        /* Turn the queue back on using our cached control register */
  93        pt_start_queue(cmd_q);
  94        mutex_unlock(&cmd_q->q_mutex);
  95
  96        return 0;
  97}
  98
  99int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
 100                             struct pt_passthru_engine *pt_engine)
 101{
 102        struct ptdma_desc desc;
 103
 104        cmd_q->cmd_error = 0;
 105        cmd_q->total_pt_ops++;
 106        memset(&desc, 0, sizeof(desc));
 107        desc.dw0 = CMD_DESC_DW0_VAL;
 108        desc.length = pt_engine->src_len;
 109        desc.src_lo = lower_32_bits(pt_engine->src_dma);
 110        desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
 111        desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
 112        desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
 113
 114        return pt_core_execute_cmd(&desc, cmd_q);
 115}
 116
 117static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
 118{
 119        iowrite32(0, pt->cmd_q.reg_control + 0x000C);
 120}
 121
 122static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
 123{
 124        iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
 125}
 126
 127static void pt_do_cmd_complete(unsigned long data)
 128{
 129        struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;
 130        struct pt_cmd *cmd = tdata->cmd;
 131        struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;
 132        u32 tail;
 133
 134        if (cmd_q->cmd_error) {
 135               /*
 136                * Log the error and flush the queue by
 137                * moving the head pointer
 138                */
 139                tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
 140                pt_log_error(cmd_q->pt, cmd_q->cmd_error);
 141                iowrite32(tail, cmd_q->reg_control + 0x0008);
 142        }
 143
 144        cmd->pt_cmd_callback(cmd->data, cmd->ret);
 145}
 146
 147static irqreturn_t pt_core_irq_handler(int irq, void *data)
 148{
 149        struct pt_device *pt = data;
 150        struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 151        u32 status;
 152
 153        pt_core_disable_queue_interrupts(pt);
 154        pt->total_interrupts++;
 155        status = ioread32(cmd_q->reg_control + 0x0010);
 156        if (status) {
 157                cmd_q->int_status = status;
 158                cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
 159                cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
 160
 161                /* On error, only save the first error value */
 162                if ((status & INT_ERROR) && !cmd_q->cmd_error)
 163                        cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
 164
 165                /* Acknowledge the interrupt */
 166                iowrite32(status, cmd_q->reg_control + 0x0010);
 167                pt_core_enable_queue_interrupts(pt);
 168                pt_do_cmd_complete((ulong)&pt->tdata);
 169        }
 170        return IRQ_HANDLED;
 171}
 172
 173int pt_core_init(struct pt_device *pt)
 174{
 175        char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
 176        struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 177        u32 dma_addr_lo, dma_addr_hi;
 178        struct device *dev = pt->dev;
 179        struct dma_pool *dma_pool;
 180        int ret;
 181
 182        /* Allocate a dma pool for the queue */
 183        snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
 184
 185        dma_pool = dma_pool_create(dma_pool_name, dev,
 186                                   PT_DMAPOOL_MAX_SIZE,
 187                                   PT_DMAPOOL_ALIGN, 0);
 188        if (!dma_pool)
 189                return -ENOMEM;
 190
 191        /* ptdma core initialisation */
 192        iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
 193        iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
 194        iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
 195        iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
 196        iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
 197
 198        cmd_q->pt = pt;
 199        cmd_q->dma_pool = dma_pool;
 200        mutex_init(&cmd_q->q_mutex);
 201
 202        /* Page alignment satisfies our needs for N <= 128 */
 203        cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
 204        cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
 205                                          &cmd_q->qbase_dma,
 206                                          GFP_KERNEL);
 207        if (!cmd_q->qbase) {
 208                dev_err(dev, "unable to allocate command queue\n");
 209                ret = -ENOMEM;
 210                goto e_dma_alloc;
 211        }
 212
 213        cmd_q->qidx = 0;
 214
 215        /* Preset some register values */
 216        cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
 217
 218        /* Turn off the queues and disable interrupts until ready */
 219        pt_core_disable_queue_interrupts(pt);
 220
 221        cmd_q->qcontrol = 0; /* Start with nothing */
 222        iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 223
 224        ioread32(cmd_q->reg_control + 0x0104);
 225        ioread32(cmd_q->reg_control + 0x0100);
 226
 227        /* Clear the interrupt status */
 228        iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
 229
 230        /* Request an irq */
 231        ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
 232        if (ret)
 233                goto e_pool;
 234
 235        /* Update the device registers with queue information. */
 236        cmd_q->qcontrol &= ~CMD_Q_SIZE;
 237        cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
 238
 239        cmd_q->qdma_tail = cmd_q->qbase_dma;
 240        dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
 241        iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
 242        iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
 243
 244        dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
 245        cmd_q->qcontrol |= (dma_addr_hi << 16);
 246        iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 247
 248        pt_core_enable_queue_interrupts(pt);
 249
 250        /* Register the DMA engine support */
 251        ret = pt_dmaengine_register(pt);
 252        if (ret)
 253                goto e_dmaengine;
 254
 255        /* Set up debugfs entries */
 256        ptdma_debugfs_setup(pt);
 257
 258        return 0;
 259
 260e_dmaengine:
 261        free_irq(pt->pt_irq, pt);
 262
 263e_dma_alloc:
 264        dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
 265
 266e_pool:
 267        dev_err(dev, "unable to allocate an IRQ\n");
 268        dma_pool_destroy(pt->cmd_q.dma_pool);
 269
 270        return ret;
 271}
 272
 273void pt_core_destroy(struct pt_device *pt)
 274{
 275        struct device *dev = pt->dev;
 276        struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 277        struct pt_cmd *cmd;
 278
 279        /* Unregister the DMA engine */
 280        pt_dmaengine_unregister(pt);
 281
 282        /* Disable and clear interrupts */
 283        pt_core_disable_queue_interrupts(pt);
 284
 285        /* Turn off the run bit */
 286        pt_stop_queue(cmd_q);
 287
 288        /* Clear the interrupt status */
 289        iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
 290        ioread32(cmd_q->reg_control + 0x0104);
 291        ioread32(cmd_q->reg_control + 0x0100);
 292
 293        free_irq(pt->pt_irq, pt);
 294
 295        dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
 296                          cmd_q->qbase_dma);
 297
 298        /* Flush the cmd queue */
 299        while (!list_empty(&pt->cmd)) {
 300                /* Invoke the callback directly with an error code */
 301                cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
 302                list_del(&cmd->entry);
 303                cmd->pt_cmd_callback(cmd->data, -ENODEV);
 304        }
 305}
 306