// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
 * Copyright (C) 2017 Linaro Ltd.
 */

#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/slab.h>

#include "core.h"
#include "hfi_cmds.h"
#include "hfi_msgs.h"
#include "hfi_venus.h"
#include "hfi_venus_io.h"
#include "firmware.h"

#define HFI_MASK_QHDR_TX_TYPE           0xff000000
#define HFI_MASK_QHDR_RX_TYPE           0x00ff0000
#define HFI_MASK_QHDR_PRI_TYPE          0x0000ff00
#define HFI_MASK_QHDR_ID_TYPE           0x000000ff

#define HFI_HOST_TO_CTRL_CMD_Q          0
#define HFI_CTRL_TO_HOST_MSG_Q          1
#define HFI_CTRL_TO_HOST_DBG_Q          2
#define HFI_MASK_QHDR_STATUS            0x000000ff

#define IFACEQ_NUM                      3
#define IFACEQ_CMD_IDX                  0
#define IFACEQ_MSG_IDX                  1
#define IFACEQ_DBG_IDX                  2
#define IFACEQ_MAX_BUF_COUNT            50
#define IFACEQ_MAX_PARALLEL_CLNTS       16
#define IFACEQ_DFLT_QHDR                0x01010000

#define POLL_INTERVAL_US                50

#define IFACEQ_MAX_PKT_SIZE             1024
#define IFACEQ_MED_PKT_SIZE             768
#define IFACEQ_MIN_PKT_SIZE             8
#define IFACEQ_VAR_SMALL_PKT_SIZE       100
#define IFACEQ_VAR_LARGE_PKT_SIZE       512
#define IFACEQ_VAR_HUGE_PKT_SIZE        (1024 * 12)

struct hfi_queue_table_header {
        u32 version;
        u32 size;
        u32 qhdr0_offset;
        u32 qhdr_size;
        u32 num_q;
        u32 num_active_q;
};

struct hfi_queue_header {
        u32 status;
        u32 start_addr;
        u32 type;
        u32 q_size;
        u32 pkt_size;
        u32 pkt_drop_cnt;
        u32 rx_wm;
        u32 tx_wm;
        u32 rx_req;
        u32 tx_req;
        u32 rx_irq_status;
        u32 tx_irq_status;
        u32 read_idx;
        u32 write_idx;
};

#define IFACEQ_TABLE_SIZE       \
        (sizeof(struct hfi_queue_table_header) +        \
         sizeof(struct hfi_queue_header) * IFACEQ_NUM)

#define IFACEQ_QUEUE_SIZE       (IFACEQ_MAX_PKT_SIZE *  \
        IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)

#define IFACEQ_GET_QHDR_START_ADDR(ptr, i)      \
        (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +      \
                ((i) * sizeof(struct hfi_queue_header)))

#define QDSS_SIZE               SZ_4K
#define SFR_SIZE                SZ_4K
#define QUEUE_SIZE              \
        (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))

#define ALIGNED_QDSS_SIZE       ALIGN(QDSS_SIZE, SZ_4K)
#define ALIGNED_SFR_SIZE        ALIGN(SFR_SIZE, SZ_4K)
#define ALIGNED_QUEUE_SIZE      ALIGN(QUEUE_SIZE, SZ_4K)
#define SHARED_QSIZE            ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
                                      ALIGNED_QDSS_SIZE, SZ_1M)

struct mem_desc {
        dma_addr_t da;  /* device address */
        void *kva;      /* kernel virtual address */
        u32 size;
        unsigned long attrs;
};

struct iface_queue {
        struct hfi_queue_header *qhdr;
        struct mem_desc qmem;
};

enum venus_state {
        VENUS_STATE_DEINIT = 1,
        VENUS_STATE_INIT,
};

struct venus_hfi_device {
        struct venus_core *core;
        u32 irq_status;
        u32 last_packet_type;
        bool power_enabled;
        bool suspended;
        enum venus_state state;
        /* serialize read / write to the shared memory */
        struct mutex lock;
        struct completion pwr_collapse_prep;
        struct completion release_resource;
        struct mem_desc ifaceq_table;
        struct mem_desc sfr;
        struct iface_queue queues[IFACEQ_NUM];
        u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
        u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
};

static bool venus_pkt_debug;
static int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
static bool venus_sys_idle_indicator;
static bool venus_fw_low_power_mode = true;
static int venus_hw_rsp_timeout = 1000;
static bool venus_fw_coverage;

static void venus_set_state(struct venus_hfi_device *hdev,
                            enum venus_state state)
{
        mutex_lock(&hdev->lock);
        hdev->state = state;
        mutex_unlock(&hdev->lock);
}

static bool venus_is_valid_state(struct venus_hfi_device *hdev)
{
        return hdev->state != VENUS_STATE_DEINIT;
}

static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
{
        size_t pkt_size = *(u32 *)packet;

        if (!venus_pkt_debug)
                return;

        print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
                       pkt_size, true);
}

static int venus_write_queue(struct venus_hfi_device *hdev,
                             struct iface_queue *queue,
                             void *packet, u32 *rx_req)
{
        struct hfi_queue_header *qhdr;
        u32 dwords, new_wr_idx;
        u32 empty_space, rd_idx, wr_idx, qsize;
        u32 *wr_ptr;

        if (!queue->qmem.kva)
                return -EINVAL;

        qhdr = queue->qhdr;
        if (!qhdr)
                return -EINVAL;

        venus_dump_packet(hdev, packet);

        dwords = (*(u32 *)packet) >> 2;
        if (!dwords)
                return -EINVAL;

        rd_idx = qhdr->read_idx;
        wr_idx = qhdr->write_idx;
        qsize = qhdr->q_size;
        /* ensure rd/wr indices's are read from memory */
        rmb();

        if (wr_idx >= rd_idx)
                empty_space = qsize - (wr_idx - rd_idx);
        else
                empty_space = rd_idx - wr_idx;

        if (empty_space <= dwords) {
                qhdr->tx_req = 1;
                /* ensure tx_req is updated in memory */
                wmb();
                return -ENOSPC;
        }

        qhdr->tx_req = 0;
        /* ensure tx_req is updated in memory */
        wmb();

        new_wr_idx = wr_idx + dwords;
        wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
        if (new_wr_idx < qsize) {
                memcpy(wr_ptr, packet, dwords << 2);
        } else {
                size_t len;

                new_wr_idx -= qsize;
                len = (dwords - new_wr_idx) << 2;
                memcpy(wr_ptr, packet, len);
                memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
        }

        /* make sure packet is written before updating the write index */
        wmb();

        qhdr->write_idx = new_wr_idx;
        *rx_req = qhdr->rx_req ? 1 : 0;

        /* make sure write index is updated before an interrupt is raised */
        mb();

        return 0;
}

static int venus_read_queue(struct venus_hfi_device *hdev,
                            struct iface_queue *queue, void *pkt, u32 *tx_req)
{
        struct hfi_queue_header *qhdr;
        u32 dwords, new_rd_idx;
        u32 rd_idx, wr_idx, type, qsize;
        u32 *rd_ptr;
        u32 recv_request = 0;
        int ret = 0;

        if (!queue->qmem.kva)
                return -EINVAL;

        qhdr = queue->qhdr;
        if (!qhdr)
                return -EINVAL;

        type = qhdr->type;
        rd_idx = qhdr->read_idx;
        wr_idx = qhdr->write_idx;
        qsize = qhdr->q_size;

        /* make sure data is valid before using it */
        rmb();

        /*
         * Do not set receive request for debug queue, if set, Venus generates
         * interrupt for debug messages even when there is no response message
         * available. In general debug queue will not become full as it is being
         * emptied out for every interrupt from Venus. Venus will anyway
         * generates interrupt if it is full.
         */
        if (type & HFI_CTRL_TO_HOST_MSG_Q)
                recv_request = 1;

        if (rd_idx == wr_idx) {
                qhdr->rx_req = recv_request;
                *tx_req = 0;
                /* update rx_req field in memory */
                wmb();
                return -ENODATA;
        }

        rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
        dwords = *rd_ptr >> 2;
        if (!dwords)
                return -EINVAL;

        new_rd_idx = rd_idx + dwords;
        if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
                if (new_rd_idx < qsize) {
                        memcpy(pkt, rd_ptr, dwords << 2);
                } else {
                        size_t len;

                        new_rd_idx -= qsize;
                        len = (dwords - new_rd_idx) << 2;
                        memcpy(pkt, rd_ptr, len);
                        memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
                }
        } else {
                /* bad packet received, dropping */
                new_rd_idx = qhdr->write_idx;
                ret = -EBADMSG;
        }

        /* ensure the packet is read before updating read index */
        rmb();

        qhdr->read_idx = new_rd_idx;
        /* ensure updating read index */
        wmb();

        rd_idx = qhdr->read_idx;
        wr_idx = qhdr->write_idx;
        /* ensure rd/wr indices are read from memory */
        rmb();

        if (rd_idx != wr_idx)
                qhdr->rx_req = 0;
        else
                qhdr->rx_req = recv_request;

        *tx_req = qhdr->tx_req ? 1 : 0;

        /* ensure rx_req is stored to memory and tx_req is loaded from memory */
        mb();

        venus_dump_packet(hdev, pkt);

        return ret;
}

static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
                       u32 size)
{
        struct device *dev = hdev->core->dev;

        desc->attrs = DMA_ATTR_WRITE_COMBINE;
        desc->size = ALIGN(size, SZ_4K);

        desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
                                    desc->attrs);
        if (!desc->kva)
                return -ENOMEM;

        return 0;
}

static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
{
        struct device *dev = hdev->core->dev;

        dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
}

static void venus_writel(struct venus_hfi_device *hdev, u32 reg, u32 value)
{
        writel(value, hdev->core->base + reg);
}

static u32 venus_readl(struct venus_hfi_device *hdev, u32 reg)
{
        return readl(hdev->core->base + reg);
}

static void venus_set_registers(struct venus_hfi_device *hdev)
{
        const struct venus_resources *res = hdev->core->res;
        const struct reg_val *tbl = res->reg_tbl;
        unsigned int count = res->reg_tbl_size;
        unsigned int i;

        for (i = 0; i < count; i++)
                venus_writel(hdev, tbl[i].reg, tbl[i].value);
}

static void venus_soft_int(struct venus_hfi_device *hdev)
{
        venus_writel(hdev, CPU_IC_SOFTINT, BIT(CPU_IC_SOFTINT_H2A_SHIFT));
}

static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
                                         void *pkt)
{
        struct device *dev = hdev->core->dev;
        struct hfi_pkt_hdr *cmd_packet;
        struct iface_queue *queue;
        u32 rx_req;
        int ret;

        if (!venus_is_valid_state(hdev))
                return -EINVAL;

        cmd_packet = (struct hfi_pkt_hdr *)pkt;
        hdev->last_packet_type = cmd_packet->pkt_type;

        queue = &hdev->queues[IFACEQ_CMD_IDX];

        ret = venus_write_queue(hdev, queue, pkt, &rx_req);
        if (ret) {
                dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
                return ret;
        }

        if (rx_req)
                venus_soft_int(hdev);

        return 0;
}

static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt)
{
        int ret;

        mutex_lock(&hdev->lock);
        ret = venus_iface_cmdq_write_nolock(hdev, pkt);
        mutex_unlock(&hdev->lock);

        return ret;
}

static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
                                       u32 size, u32 addr, void *cookie)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct hfi_sys_set_resource_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        if (id == VIDC_RESOURCE_NONE)
                return 0;

        pkt = (struct hfi_sys_set_resource_pkt *)packet;

        ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
        if (ret)
                return ret;

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_boot_core(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        static const unsigned int max_tries = 100;
        u32 ctrl_status = 0;
        unsigned int count = 0;
        int ret = 0;

        venus_writel(hdev, VIDC_CTRL_INIT, BIT(VIDC_CTRL_INIT_CTRL_SHIFT));
        venus_writel(hdev, WRAPPER_INTR_MASK, WRAPPER_INTR_MASK_A2HVCODEC_MASK);
        venus_writel(hdev, CPU_CS_SCIACMDARG3, 1);

        while (!ctrl_status && count < max_tries) {
                ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
                if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
                        dev_err(dev, "invalid setting for UC_REGION\n");
                        ret = -EINVAL;
                        break;
                }

                usleep_range(500, 1000);
                count++;
        }

        if (count >= max_tries)
                ret = -ETIMEDOUT;

        return ret;
}

static u32 venus_hwversion(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        u32 ver = venus_readl(hdev, WRAPPER_HW_VERSION);
        u32 major, minor, step;

        major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
        major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
        minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
        minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
        step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;

        dev_dbg(dev, "venus hw version %x.%x.%x\n", major, minor, step);

        return major;
}

static int venus_run(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        int ret;

        /*
         * Re-program all of the registers that get reset as a result of
         * regulator_disable() and _enable()
         */
        venus_set_registers(hdev);

        venus_writel(hdev, UC_REGION_ADDR, hdev->ifaceq_table.da);
        venus_writel(hdev, UC_REGION_SIZE, SHARED_QSIZE);
        venus_writel(hdev, CPU_CS_SCIACMDARG2, hdev->ifaceq_table.da);
        venus_writel(hdev, CPU_CS_SCIACMDARG1, 0x01);
        if (hdev->sfr.da)
                venus_writel(hdev, SFR_ADDR, hdev->sfr.da);

        ret = venus_boot_core(hdev);
        if (ret) {
                dev_err(dev, "failed to reset venus core\n");
                return ret;
        }

        venus_hwversion(hdev);

        return 0;
}

static int venus_halt_axi(struct venus_hfi_device *hdev)
{
        void __iomem *base = hdev->core->base;
        struct device *dev = hdev->core->dev;
        u32 val;
        int ret;

        if (IS_V4(hdev->core)) {
                val = venus_readl(hdev, WRAPPER_CPU_AXI_HALT);
                val |= WRAPPER_CPU_AXI_HALT_HALT;
                venus_writel(hdev, WRAPPER_CPU_AXI_HALT, val);

                ret = readl_poll_timeout(base + WRAPPER_CPU_AXI_HALT_STATUS,
                                         val,
                                         val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
                                         POLL_INTERVAL_US,
                                         VBIF_AXI_HALT_ACK_TIMEOUT_US);
                if (ret) {
                        dev_err(dev, "AXI bus port halt timeout\n");
                        return ret;
                }

                return 0;
        }

        /* Halt AXI and AXI IMEM VBIF Access */
        val = venus_readl(hdev, VBIF_AXI_HALT_CTRL0);
        val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
        venus_writel(hdev, VBIF_AXI_HALT_CTRL0, val);

        /* Request for AXI bus port halt */
        ret = readl_poll_timeout(base + VBIF_AXI_HALT_CTRL1, val,
                                 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
                                 POLL_INTERVAL_US,
                                 VBIF_AXI_HALT_ACK_TIMEOUT_US);
        if (ret) {
                dev_err(dev, "AXI bus port halt timeout\n");
                return ret;
        }

        return 0;
}

static int venus_power_off(struct venus_hfi_device *hdev)
{
        int ret;

        if (!hdev->power_enabled)
                return 0;

        ret = venus_set_hw_state_suspend(hdev->core);
        if (ret)
                return ret;

        ret = venus_halt_axi(hdev);
        if (ret)
                return ret;

        hdev->power_enabled = false;

        return 0;
}

static int venus_power_on(struct venus_hfi_device *hdev)
{
        int ret;

        if (hdev->power_enabled)
                return 0;

        ret = venus_set_hw_state_resume(hdev->core);
        if (ret)
                goto err;

        ret = venus_run(hdev);
        if (ret)
                goto err_suspend;

        hdev->power_enabled = true;

        return 0;

err_suspend:
        venus_set_hw_state_suspend(hdev->core);
err:
        hdev->power_enabled = false;
        return ret;
}

static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
                                        void *pkt)
{
        struct iface_queue *queue;
        u32 tx_req;
        int ret;

        if (!venus_is_valid_state(hdev))
                return -EINVAL;

        queue = &hdev->queues[IFACEQ_MSG_IDX];

        ret = venus_read_queue(hdev, queue, pkt, &tx_req);
        if (ret)
                return ret;

        if (tx_req)
                venus_soft_int(hdev);

        return 0;
}

static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
{
        int ret;

        mutex_lock(&hdev->lock);
        ret = venus_iface_msgq_read_nolock(hdev, pkt);
        mutex_unlock(&hdev->lock);

        return ret;
}

static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
                                        void *pkt)
{
        struct iface_queue *queue;
        u32 tx_req;
        int ret;

        ret = venus_is_valid_state(hdev);
        if (!ret)
                return -EINVAL;

        queue = &hdev->queues[IFACEQ_DBG_IDX];

        ret = venus_read_queue(hdev, queue, pkt, &tx_req);
        if (ret)
                return ret;

        if (tx_req)
                venus_soft_int(hdev);

        return 0;
}

static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
{
        int ret;

        if (!pkt)
                return -EINVAL;

        mutex_lock(&hdev->lock);
        ret = venus_iface_dbgq_read_nolock(hdev, pkt);
        mutex_unlock(&hdev->lock);

        return ret;
}

static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
{
        qhdr->status = 1;
        qhdr->type = IFACEQ_DFLT_QHDR;
        qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
        qhdr->pkt_size = 0;
        qhdr->rx_wm = 1;
        qhdr->tx_wm = 1;
        qhdr->rx_req = 1;
        qhdr->tx_req = 0;
        qhdr->rx_irq_status = 0;
        qhdr->tx_irq_status = 0;
        qhdr->read_idx = 0;
        qhdr->write_idx = 0;
}

static void venus_interface_queues_release(struct venus_hfi_device *hdev)
{
        mutex_lock(&hdev->lock);

        venus_free(hdev, &hdev->ifaceq_table);
        venus_free(hdev, &hdev->sfr);

        memset(hdev->queues, 0, sizeof(hdev->queues));
        memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
        memset(&hdev->sfr, 0, sizeof(hdev->sfr));

        mutex_unlock(&hdev->lock);
}

static int venus_interface_queues_init(struct venus_hfi_device *hdev)
{
        struct hfi_queue_table_header *tbl_hdr;
        struct iface_queue *queue;
        struct hfi_sfr *sfr;
        struct mem_desc desc = {0};
        unsigned int offset;
        unsigned int i;
        int ret;

        ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
        if (ret)
                return ret;

        hdev->ifaceq_table = desc;
        offset = IFACEQ_TABLE_SIZE;

        for (i = 0; i < IFACEQ_NUM; i++) {
                queue = &hdev->queues[i];
                queue->qmem.da = desc.da + offset;
                queue->qmem.kva = desc.kva + offset;
                queue->qmem.size = IFACEQ_QUEUE_SIZE;
                offset += queue->qmem.size;
                queue->qhdr =
                        IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);

                venus_set_qhdr_defaults(queue->qhdr);

                queue->qhdr->start_addr = queue->qmem.da;

                if (i == IFACEQ_CMD_IDX)
                        queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
                else if (i == IFACEQ_MSG_IDX)
                        queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
                else if (i == IFACEQ_DBG_IDX)
                        queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
        }

        tbl_hdr = hdev->ifaceq_table.kva;
        tbl_hdr->version = 0;
        tbl_hdr->size = IFACEQ_TABLE_SIZE;
        tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
        tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
        tbl_hdr->num_q = IFACEQ_NUM;
        tbl_hdr->num_active_q = IFACEQ_NUM;

        /*
         * Set receive request to zero on debug queue as there is no
         * need of interrupt from video hardware for debug messages
         */
        queue = &hdev->queues[IFACEQ_DBG_IDX];
        queue->qhdr->rx_req = 0;

        ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
        if (ret) {
                hdev->sfr.da = 0;
        } else {
                hdev->sfr = desc;
                sfr = hdev->sfr.kva;
                sfr->buf_size = ALIGNED_SFR_SIZE;
        }

        /* ensure table and queue header structs are settled in memory */
        wmb();

        return 0;
}

static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
{
        struct hfi_sys_set_property_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        pkt = (struct hfi_sys_set_property_pkt *)packet;

        pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
{
        struct hfi_sys_set_property_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        pkt = (struct hfi_sys_set_property_pkt *)packet;

        pkt_sys_coverage_config(pkt, mode);

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
                                      bool enable)
{
        struct hfi_sys_set_property_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        if (!enable)
                return 0;

        pkt = (struct hfi_sys_set_property_pkt *)packet;

        pkt_sys_idle_indicator(pkt, enable);

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
                                       bool enable)
{
        struct hfi_sys_set_property_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        pkt = (struct hfi_sys_set_property_pkt *)packet;

        pkt_sys_power_control(pkt, enable);

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_get_queue_size(struct venus_hfi_device *hdev,
                                unsigned int index)
{
        struct hfi_queue_header *qhdr;

        if (index >= IFACEQ_NUM)
                return -EINVAL;

        qhdr = hdev->queues[index].qhdr;
        if (!qhdr)
                return -EINVAL;

        return abs(qhdr->read_idx - qhdr->write_idx);
}

static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        int ret;

        ret = venus_sys_set_debug(hdev, venus_fw_debug);
        if (ret)
                dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);

        /*
         * Idle indicator is disabled by default on some 4xx firmware versions,
         * enable it explicitly in order to make suspend functional by checking
         * WFI (wait-for-interrupt) bit.
         */
        if (IS_V4(hdev->core))
                venus_sys_idle_indicator = true;

        ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
        if (ret)
                dev_warn(dev, "setting idle response ON failed (%d)\n", ret);

        ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
        if (ret)
                dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
                         ret);

        return ret;
}

static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_pkt pkt;

        pkt_session_cmd(&pkt, pkt_type, inst);

        return venus_iface_cmdq_write(hdev, &pkt);
}

static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        void *packet = hdev->dbg_buf;

        while (!venus_iface_dbgq_read(hdev, packet)) {
                struct hfi_msg_sys_coverage_pkt *pkt = packet;

                if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
                        struct hfi_msg_sys_debug_pkt *pkt = packet;

                        dev_dbg(dev, "%s", pkt->msg_data);
                }
        }
}

static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
                                        bool wait)
{
        unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
        struct hfi_sys_pc_prep_pkt pkt;
        int ret;

        init_completion(&hdev->pwr_collapse_prep);

        pkt_sys_pc_prep(&pkt);

        ret = venus_iface_cmdq_write(hdev, &pkt);
        if (ret)
                return ret;

        if (!wait)
                return 0;

        ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
        if (!ret) {
                venus_flush_debug_queue(hdev);
                return -ETIMEDOUT;
        }

        return 0;
}

static int venus_are_queues_empty(struct venus_hfi_device *hdev)
{
        int ret1, ret2;

        ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
        if (ret1 < 0)
                return ret1;

        ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
        if (ret2 < 0)
                return ret2;

        if (!ret1 && !ret2)
                return 1;

        return 0;
}

static void venus_sfr_print(struct venus_hfi_device *hdev)
{
        struct device *dev = hdev->core->dev;
        struct hfi_sfr *sfr = hdev->sfr.kva;
        void *p;

        if (!sfr)
                return;

        p = memchr(sfr->data, '\0', sfr->buf_size);
        /*
         * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
         * that Venus is in the process of crashing.
         */
        if (!p)
                sfr->data[sfr->buf_size - 1] = '\0';

        dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
}

static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
                                        void *packet)
{
        struct hfi_msg_event_notify_pkt *event_pkt = packet;

        if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
                return;

        venus_set_state(hdev, VENUS_STATE_DEINIT);

        /*
         * Once SYS_ERROR received from HW, it is safe to halt the AXI.
         * With SYS_ERROR, Venus FW may have crashed and HW might be
         * active and causing unnecessary transactions. Hence it is
         * safe to stop all AXI transactions from venus subsystem.
         */
        venus_halt_axi(hdev);
        venus_sfr_print(hdev);
}

static irqreturn_t venus_isr_thread(struct venus_core *core)
{

        struct venus_hfi_device *hdev = to_hfi_priv(core);
        const struct venus_resources *res;
        void *pkt;
        u32 msg_ret;

        if (!hdev)
                return IRQ_NONE;

        res = hdev->core->res;
        pkt = hdev->pkt_buf;

        if (hdev->irq_status & WRAPPER_INTR_STATUS_A2HWD_MASK) {
                venus_sfr_print(hdev);
                hfi_process_watchdog_timeout(core);
        }

        while (!venus_iface_msgq_read(hdev, pkt)) {
                msg_ret = hfi_process_msg_packet(core, pkt);
                switch (msg_ret) {
                case HFI_MSG_EVENT_NOTIFY:
                        venus_process_msg_sys_error(hdev, pkt);
                        break;
                case HFI_MSG_SYS_INIT:
                        venus_hfi_core_set_resource(core, res->vmem_id,
                                                    res->vmem_size,
                                                    res->vmem_addr,
                                                    hdev);
                        break;
                case HFI_MSG_SYS_RELEASE_RESOURCE:
                        complete(&hdev->release_resource);
                        break;
                case HFI_MSG_SYS_PC_PREP:
                        complete(&hdev->pwr_collapse_prep);
                        break;
                default:
                        break;
                }
        }

        venus_flush_debug_queue(hdev);

        return IRQ_HANDLED;
}

static irqreturn_t venus_isr(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        u32 status;

        if (!hdev)
                return IRQ_NONE;

        status = venus_readl(hdev, WRAPPER_INTR_STATUS);

        if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
            status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
            status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
                hdev->irq_status = status;

        venus_writel(hdev, CPU_CS_A2HSOFTINTCLR, 1);
        venus_writel(hdev, WRAPPER_INTR_CLEAR, status);

        return IRQ_WAKE_THREAD;
}

static int venus_core_init(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct device *dev = core->dev;
        struct hfi_sys_get_property_pkt version_pkt;
        struct hfi_sys_init_pkt pkt;
        int ret;

        pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);

        venus_set_state(hdev, VENUS_STATE_INIT);

        ret = venus_iface_cmdq_write(hdev, &pkt);
        if (ret)
                return ret;

        pkt_sys_image_version(&version_pkt);

        ret = venus_iface_cmdq_write(hdev, &version_pkt);
        if (ret)
                dev_warn(dev, "failed to send image version pkt to fw\n");

        ret = venus_sys_set_default_properties(hdev);
        if (ret)
                return ret;

        return 0;
}

static int venus_core_deinit(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);

        venus_set_state(hdev, VENUS_STATE_DEINIT);
        hdev->suspended = true;
        hdev->power_enabled = false;

        return 0;
}

static int venus_core_ping(struct venus_core *core, u32 cookie)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct hfi_sys_ping_pkt pkt;

        pkt_sys_ping(&pkt, cookie);

        return venus_iface_cmdq_write(hdev, &pkt);
}

static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct hfi_sys_test_ssr_pkt pkt;
        int ret;

        ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, &pkt);
}

static int venus_session_init(struct venus_inst *inst, u32 session_type,
                              u32 codec)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_init_pkt pkt;
        int ret;

        ret = pkt_session_init(&pkt, inst, session_type, codec);
        if (ret)
                goto err;

        ret = venus_iface_cmdq_write(hdev, &pkt);
        if (ret)
                goto err;

        return 0;

err:
        venus_flush_debug_queue(hdev);
        return ret;
}

static int venus_session_end(struct venus_inst *inst)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct device *dev = hdev->core->dev;

        if (venus_fw_coverage) {
                if (venus_sys_set_coverage(hdev, venus_fw_coverage))
                        dev_warn(dev, "fw coverage msg ON failed\n");
        }

        return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END);
}

static int venus_session_abort(struct venus_inst *inst)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);

        venus_flush_debug_queue(hdev);

        return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT);
}

static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_flush_pkt pkt;
        int ret;

        ret = pkt_session_flush(&pkt, inst, flush_mode);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, &pkt);
}

static int venus_session_start(struct venus_inst *inst)
{
        return venus_session_cmd(inst, HFI_CMD_SESSION_START);
}

static int venus_session_stop(struct venus_inst *inst)
{
        return venus_session_cmd(inst, HFI_CMD_SESSION_STOP);
}

static int venus_session_continue(struct venus_inst *inst)
{
        return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE);
}

static int venus_session_etb(struct venus_inst *inst,
                             struct hfi_frame_data *in_frame)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        u32 session_type = inst->session_type;
        int ret;

        if (session_type == VIDC_SESSION_TYPE_DEC) {
                struct hfi_session_empty_buffer_compressed_pkt pkt;

                ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
                if (ret)
                        return ret;

                ret = venus_iface_cmdq_write(hdev, &pkt);
        } else if (session_type == VIDC_SESSION_TYPE_ENC) {
                struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;

                ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
                if (ret)
                        return ret;

                ret = venus_iface_cmdq_write(hdev, &pkt);
        } else {
                ret = -EINVAL;
        }

        return ret;
}

static int venus_session_ftb(struct venus_inst *inst,
                             struct hfi_frame_data *out_frame)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_fill_buffer_pkt pkt;
        int ret;

        ret = pkt_session_ftb(&pkt, inst, out_frame);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, &pkt);
}

static int venus_session_set_buffers(struct venus_inst *inst,
                                     struct hfi_buffer_desc *bd)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_set_buffers_pkt *pkt;
        u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
        int ret;

        if (bd->buffer_type == HFI_BUFFER_INPUT)
                return 0;

        pkt = (struct hfi_session_set_buffers_pkt *)packet;

        ret = pkt_session_set_buffers(pkt, inst, bd);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, pkt);
}

static int venus_session_unset_buffers(struct venus_inst *inst,
                                       struct hfi_buffer_desc *bd)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_release_buffer_pkt *pkt;
        u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
        int ret;

        if (bd->buffer_type == HFI_BUFFER_INPUT)
                return 0;

        pkt = (struct hfi_session_release_buffer_pkt *)packet;

        ret = pkt_session_unset_buffers(pkt, inst, bd);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, pkt);
}

static int venus_session_load_res(struct venus_inst *inst)
{
        return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES);
}

static int venus_session_release_res(struct venus_inst *inst)
{
        return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES);
}

static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
                                       u32 seq_hdr_len)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_parse_sequence_header_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;

        ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
        if (ret)
                return ret;

        ret = venus_iface_cmdq_write(hdev, pkt);
        if (ret)
                return ret;

        return 0;
}

static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
                                     u32 seq_hdr_len)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_get_sequence_header_pkt *pkt;
        u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
        int ret;

        pkt = (struct hfi_session_get_sequence_header_pkt *)packet;

        ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, pkt);
}

static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
                                      void *pdata)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_set_property_pkt *pkt;
        u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
        int ret;

        pkt = (struct hfi_session_set_property_pkt *)packet;

        ret = pkt_session_set_property(pkt, inst, ptype, pdata);
        if (ret == -ENOTSUPP)
                return 0;
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, pkt);
}

static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
{
        struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
        struct hfi_session_get_property_pkt pkt;
        int ret;

        ret = pkt_session_get_property(&pkt, inst, ptype);
        if (ret)
                return ret;

        return venus_iface_cmdq_write(hdev, &pkt);
}

static int venus_resume(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        int ret = 0;

        mutex_lock(&hdev->lock);

        if (!hdev->suspended)
                goto unlock;

        ret = venus_power_on(hdev);

unlock:
        if (!ret)
                hdev->suspended = false;

        mutex_unlock(&hdev->lock);

        return ret;
}

static int venus_suspend_1xx(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct device *dev = core->dev;
        u32 ctrl_status;
        int ret;

        if (!hdev->power_enabled || hdev->suspended)
                return 0;

        mutex_lock(&hdev->lock);
        ret = venus_is_valid_state(hdev);
        mutex_unlock(&hdev->lock);

        if (!ret) {
                dev_err(dev, "bad state, cannot suspend\n");
                return -EINVAL;
        }

        ret = venus_prepare_power_collapse(hdev, true);
        if (ret) {
                dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
                return ret;
        }

        mutex_lock(&hdev->lock);

        if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
                mutex_unlock(&hdev->lock);
                return -EINVAL;
        }

        ret = venus_are_queues_empty(hdev);
        if (ret < 0 || !ret) {
                mutex_unlock(&hdev->lock);
                return -EINVAL;
        }

        ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
        if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
                mutex_unlock(&hdev->lock);
                return -EINVAL;
        }

        ret = venus_power_off(hdev);
        if (ret) {
                mutex_unlock(&hdev->lock);
                return ret;
        }

        hdev->suspended = true;

        mutex_unlock(&hdev->lock);

        return 0;
}

static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
{
        u32 ctrl_status, cpu_status;

        cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
        ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);

        if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
            ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
                return true;

        return false;
}

static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
{
        u32 ctrl_status, cpu_status;

        cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
        ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);

        if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
            ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
                return true;

        return false;
}

static int venus_suspend_3xx(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);
        struct device *dev = core->dev;
        bool val;
        int ret;

        if (!hdev->power_enabled || hdev->suspended)
                return 0;

        mutex_lock(&hdev->lock);
        ret = venus_is_valid_state(hdev);
        mutex_unlock(&hdev->lock);

        if (!ret) {
                dev_err(dev, "bad state, cannot suspend\n");
                return -EINVAL;
        }

        /*
         * Power collapse sequence for Venus 3xx and 4xx versions:
         * 1. Check for ARM9 and video core to be idle by checking WFI bit
         *    (bit 0) in CPU status register and by checking Idle (bit 30) in
         *    Control status register for video core.
         * 2. Send a command to prepare for power collapse.
         * 3. Check for WFI and PC_READY bits.
         */
        ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
                                 1500, 100 * 1500);
        if (ret)
                return ret;

        ret = venus_prepare_power_collapse(hdev, false);
        if (ret) {
                dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
                return ret;
        }

        ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
                                 1500, 100 * 1500);
        if (ret)
                return ret;

        mutex_lock(&hdev->lock);

        ret = venus_power_off(hdev);
        if (ret) {
                dev_err(dev, "venus_power_off (%d)\n", ret);
                mutex_unlock(&hdev->lock);
                return ret;
        }

        hdev->suspended = true;

        mutex_unlock(&hdev->lock);

        return 0;
}

static int venus_suspend(struct venus_core *core)
{
        if (IS_V3(core) || IS_V4(core))
                return venus_suspend_3xx(core);

        return venus_suspend_1xx(core);
}

static const struct hfi_ops venus_hfi_ops = {
        .core_init                      = venus_core_init,
        .core_deinit                    = venus_core_deinit,
        .core_ping                      = venus_core_ping,
        .core_trigger_ssr               = venus_core_trigger_ssr,

        .session_init                   = venus_session_init,
        .session_end                    = venus_session_end,
        .session_abort                  = venus_session_abort,
        .session_flush                  = venus_session_flush,
        .session_start                  = venus_session_start,
        .session_stop                   = venus_session_stop,
        .session_continue               = venus_session_continue,
        .session_etb                    = venus_session_etb,
        .session_ftb                    = venus_session_ftb,
        .session_set_buffers            = venus_session_set_buffers,
        .session_unset_buffers          = venus_session_unset_buffers,
        .session_load_res               = venus_session_load_res,
        .session_release_res            = venus_session_release_res,
        .session_parse_seq_hdr          = venus_session_parse_seq_hdr,
        .session_get_seq_hdr            = venus_session_get_seq_hdr,
        .session_set_property           = venus_session_set_property,
        .session_get_property           = venus_session_get_property,

        .resume                         = venus_resume,
        .suspend                        = venus_suspend,

        .isr                            = venus_isr,
        .isr_thread                     = venus_isr_thread,
};

void venus_hfi_destroy(struct venus_core *core)
{
        struct venus_hfi_device *hdev = to_hfi_priv(core);

        venus_interface_queues_release(hdev);
        mutex_destroy(&hdev->lock);
        kfree(hdev);
        core->priv = NULL;
        core->ops = NULL;
}

int venus_hfi_create(struct venus_core *core)
{
        struct venus_hfi_device *hdev;
        int ret;

        hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
        if (!hdev)
                return -ENOMEM;

        mutex_init(&hdev->lock);

        hdev->core = core;
        hdev->suspended = true;
        core->priv = hdev;
        core->ops = &venus_hfi_ops;
        core->core_caps = ENC_ROTATION_CAPABILITY | ENC_SCALING_CAPABILITY |
                          ENC_DEINTERLACE_CAPABILITY |
                          DEC_MULTI_STREAM_CAPABILITY;

        ret = venus_interface_queues_init(hdev);
        if (ret)
                goto err_kfree;

        return 0;

err_kfree:
        kfree(hdev);
        core->priv = NULL;
        core->ops = NULL;
        return ret;
}