/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2017 Huawei Technologies Co., Ltd
 */

#include "hinic_compat.h"
#include "hinic_csr.h"
#include "hinic_pmd_hwdev.h"
#include "hinic_pmd_cmd.h"
#include "hinic_pmd_hwif.h"
#include "hinic_pmd_api_cmd.h"

#define API_CMD_CHAIN_CELL_SIZE_SHIFT   6U

#define API_CMD_CELL_DESC_SIZE          8
#define API_CMD_CELL_DATA_ADDR_SIZE     8

#define API_CHAIN_NUM_CELLS             32
#define API_CHAIN_CELL_SIZE             128
#define API_CHAIN_RSP_DATA_SIZE         128

#define API_CHAIN_CELL_ALIGNMENT        8

#define API_CMD_TIMEOUT                 10000

#define API_CMD_BUF_SIZE                2048UL

#define API_CMD_NODE_ALIGN_SIZE         512UL
#define API_PAYLOAD_ALIGN_SIZE          64

#define API_CHAIN_RESP_ALIGNMENT        64ULL

#define COMPLETION_TIMEOUT_DEFAULT              1000UL
#define POLLING_COMPLETION_TIMEOUT_DEFAULT      1000U

#define API_CMD_RESPONSE_DATA_PADDR(val)        be64_to_cpu(*((u64 *)(val)))

#define READ_API_CMD_PRIV_DATA(id, token)       (((id) << 16) + (token))
#define WRITE_API_CMD_PRIV_DATA(id)             ((id) << 16)

#define MASKED_IDX(chain, idx)          ((idx) & ((chain)->num_cells - 1))

#undef  SIZE_4BYTES
#undef  SIZE_8BYTES
#define SIZE_4BYTES(size)               (ALIGN((u32)(size), 4U) >> 2)
#define SIZE_8BYTES(size)               (ALIGN((u32)(size), 8U) >> 3)

enum api_cmd_data_format {
        SGL_DATA     = 1,
};

enum api_cmd_type {
        API_CMD_WRITE_TYPE = 0,
        API_CMD_READ_TYPE = 1,
};

enum api_cmd_bypass {
        NOT_BYPASS = 0,
        BYPASS = 1,
};

enum api_cmd_resp_aeq {
        NOT_TRIGGER = 0,
        TRIGGER     = 1,
};

static u8 xor_chksum_set(void *data)
{
        int idx;
        u8 checksum = 0;
        u8 *val = (u8 *)data;

        for (idx = 0; idx < 7; idx++)
                checksum ^= val[idx];

        return checksum;
}

static void set_prod_idx(struct hinic_api_cmd_chain *chain)
{
        enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 hw_prod_idx_addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
        u32 prod_idx = chain->prod_idx;

        hinic_hwif_write_reg(hwif, hw_prod_idx_addr, prod_idx);
}

static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
{
        u32 addr, val;

        addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
        val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);

        return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
}

static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain)
{
        u32 addr, val;

        addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
        val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);

        PMD_DRV_LOG(ERR, "chain type: 0x%x", chain->chain_type);
        PMD_DRV_LOG(ERR, "chain hw cpld error: 0x%x",
                HINIC_API_CMD_STATUS_GET(val, CPLD_ERR));
        PMD_DRV_LOG(ERR, "chain hw check error: 0x%x",
                HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR));
        PMD_DRV_LOG(ERR, "chain hw current fsm: 0x%x",
                HINIC_API_CMD_STATUS_GET(val, FSM));
        PMD_DRV_LOG(ERR, "chain hw current ci: 0x%x",
                HINIC_API_CMD_STATUS_GET(val, CONS_IDX));

        addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type);
        val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);
        PMD_DRV_LOG(ERR, "Chain hw current pi: 0x%x", val);
}

/**
 * chain_busy - check if the chain is still processing last requests
 * @chain: chain to check
 */
static int chain_busy(struct hinic_api_cmd_chain *chain)
{
        switch (chain->chain_type) {
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                chain->cons_idx = get_hw_cons_idx(chain);
                if (chain->cons_idx == MASKED_IDX(chain, chain->prod_idx + 1)) {
                        PMD_DRV_LOG(ERR, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d",
                                chain->chain_type, chain->cons_idx,
                                chain->prod_idx);
                        dump_api_chain_reg(chain);
                        return -EBUSY;
                }
                break;
        default:
                PMD_DRV_LOG(ERR, "Unknown Chain type");
                return -EINVAL;
        }

        return 0;
}

/**
 * get_cell_data_size - get the data size of specific cell type
 * @type: chain type
 */
static u16 get_cell_data_size(enum hinic_api_cmd_chain_type type,
                                __rte_unused u16 cmd_size)
{
        u16 cell_data_size = 0;

        switch (type) {
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
                                        API_CMD_CELL_DATA_ADDR_SIZE,
                                        API_CHAIN_CELL_ALIGNMENT);
                break;
        default:
                break;
        }

        return cell_data_size;
}

/**
 * prepare_cell_ctrl - prepare the ctrl of the cell for the command
 * @cell_ctrl: the control of the cell to set the control into it
 * @cell_len: the size of the cell
 */
static void prepare_cell_ctrl(u64 *cell_ctrl, u16 cell_len)
{
        u64 ctrl;
        u8 chksum;

        /* Read Modify Write */
        ctrl = be64_to_cpu(*cell_ctrl);
        ctrl = HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, CELL_LEN) &
                HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, RD_DMA_ATTR_OFF) &
                HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, WR_DMA_ATTR_OFF) &
                HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, XOR_CHKSUM);

        ctrl |=  HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(cell_len), CELL_LEN) |
                HINIC_API_CMD_CELL_CTRL_SET(0ULL, RD_DMA_ATTR_OFF) |
                HINIC_API_CMD_CELL_CTRL_SET(0ULL, WR_DMA_ATTR_OFF);

        chksum = xor_chksum_set(&ctrl);

        ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);

        /* The data in the HW should be in Big Endian Format */
        *cell_ctrl = cpu_to_be64(ctrl);
}

/**
 * prepare_api_cmd - prepare API CMD command
 * @chain: chain for the command
 * @cell: the cell of the command
 * @dest: destination node on the card that will receive the command
 * @cmd: command data
 * @cmd_size: the command size
 */
static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
                                struct hinic_api_cmd_cell *cell,
                                enum hinic_node_id dest,
                                void *cmd, u16 cmd_size)
{
        struct hinic_api_cmd_cell_ctxt  *cell_ctxt;
        u32 priv;

        cell_ctxt = &chain->cell_ctxt[chain->prod_idx];

        /* Clear all the members before changes */
        cell->desc = HINIC_API_CMD_DESC_CLEAR(cell->desc, API_TYPE) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, RD_WR) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, MGMT_BYPASS) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, RESP_AEQE_EN) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, DEST) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, SIZE) &
                        HINIC_API_CMD_DESC_CLEAR(cell->desc, XOR_CHKSUM);

        switch (chain->chain_type) {
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                priv =  WRITE_API_CMD_PRIV_DATA(chain->chain_type);
                cell->desc = HINIC_API_CMD_DESC_SET(SGL_DATA, API_TYPE) |
                        HINIC_API_CMD_DESC_SET(API_CMD_WRITE_TYPE, RD_WR) |
                        HINIC_API_CMD_DESC_SET(NOT_BYPASS, MGMT_BYPASS) |
                        HINIC_API_CMD_DESC_SET(TRIGGER, RESP_AEQE_EN)   |
                        HINIC_API_CMD_DESC_SET(priv, PRIV_DATA);
                break;
        default:
                PMD_DRV_LOG(ERR, "Unknown Chain type");
                return;
        }

        cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST)         |
                        HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
        cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
                                                XOR_CHKSUM);

        /* The data in the HW should be in Big Endian Format */
        cell->desc = cpu_to_be64(cell->desc);

        memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
}

/**
 * prepare_cell - prepare cell ctrl and cmd in the current producer cell
 * @chain: chain for the command
 * @dest: destination node on the card that will receive the command
 * @cmd: command data
 * @cmd_size: the command size
 */
static void prepare_cell(struct hinic_api_cmd_chain *chain,
                         enum  hinic_node_id dest,
                         void *cmd, u16 cmd_size)
{
        struct hinic_api_cmd_cell *curr_node;
        u16 cell_size;

        curr_node = chain->curr_node;

        cell_size = get_cell_data_size(chain->chain_type, cmd_size);

        prepare_cell_ctrl(&curr_node->ctrl, cell_size);
        prepare_api_cmd(chain, curr_node, dest, cmd, cmd_size);
}

static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
{
        chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
}

static void issue_api_cmd(struct hinic_api_cmd_chain *chain)
{
        set_prod_idx(chain);
}

/**
 * api_cmd_status_update - update the status of the chain
 * @chain: chain to update
 */
static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
{
        struct hinic_api_cmd_status *wb_status;
        enum hinic_api_cmd_chain_type chain_type;
        u64     status_header;
        u32     buf_desc;

        wb_status = chain->wb_status;

        buf_desc = be32_to_cpu(wb_status->buf_desc);
        if (HINIC_API_CMD_STATUS_GET(buf_desc, CHKSUM_ERR)) {
                PMD_DRV_LOG(ERR, "API CMD status Xor check error");
                return;
        }

        status_header = be64_to_cpu(wb_status->header);
        chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
        if (chain_type >= HINIC_API_CMD_MAX)
                return;

        if (chain_type != chain->chain_type)
                return;

        chain->cons_idx = HINIC_API_CMD_STATUS_GET(buf_desc, CONS_IDX);
}

/**
 * wait_for_status_poll - wait for write to mgmt command to complete
 * @chain: the chain of the command
 * Return: 0 - success, negative - failure
 */
static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
{
        unsigned long end;
        int err = -ETIMEDOUT;

        end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
        do {
                api_cmd_status_update(chain);

                /* SYNC API CMD cmd should start after prev cmd finished */
                if (chain->cons_idx == chain->prod_idx) {
                        err = 0;
                        break;
                }

                rte_delay_us(10);
        } while (time_before(jiffies, end));

        return err;
}

/**
 * wait_for_api_cmd_completion - wait for command to complete
 * @chain: chain for the command
 * Return: 0 - success, negative - failure
 */
static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain,
                       __rte_unused struct hinic_api_cmd_cell_ctxt *ctxt,
                       __rte_unused void *ack, __rte_unused u16 ack_size)
{
        int err = 0;

        /* poll api cmd status for debug*/
        switch (chain->chain_type) {
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                err = wait_for_status_poll(chain);
                if (err)
                        PMD_DRV_LOG(ERR, "API CMD poll status timeout");
                break;
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
                break;
        default:
                PMD_DRV_LOG(ERR, "Unknown API CMD chain type");
                err = -EINVAL;
                break;
        }

        if (err)
                dump_api_chain_reg(chain);

        return err;
}

static inline void update_api_cmd_ctxt(struct hinic_api_cmd_chain *chain,
                                       struct hinic_api_cmd_cell_ctxt *ctxt)
{
        ctxt->status = 1;
        ctxt->saved_prod_idx = chain->prod_idx;
}

/**
 * api_cmd - API CMD command
 * @chain: chain for the command
 * @dest: destination node on the card that will receive the command
 * @cmd: command data
 * @cmd_size: the command size
 * @ack: pointer to messages to response
 * @ack_size: the size of ack message
 * Return: 0 - success, negative - failure
 */
static int api_cmd(struct hinic_api_cmd_chain *chain,
                   enum hinic_node_id dest,
                   void *cmd, u16 cmd_size, void *ack, u16 ack_size)
{
        struct hinic_api_cmd_cell_ctxt *ctxt;

        spin_lock(&chain->async_lock);

        ctxt = &chain->cell_ctxt[chain->prod_idx];
        if (chain_busy(chain)) {
                spin_unlock(&chain->async_lock);
                return -EBUSY;
        }
        update_api_cmd_ctxt(chain, ctxt);

        prepare_cell(chain, dest, cmd, cmd_size);

        cmd_chain_prod_idx_inc(chain);

        rte_wmb();/* issue the command */

        issue_api_cmd(chain);

        /* incremented prod idx, update ctxt */
        chain->curr_node = chain->cell_ctxt[chain->prod_idx].cell_vaddr;

        spin_unlock(&chain->async_lock);

        return wait_for_api_cmd_completion(chain, ctxt, ack, ack_size);
}

/**
 * hinic_api_cmd_write - Write API CMD command
 * @chain: chain for write command
 * @dest: destination node on the card that will receive the command
 * @cmd: command data
 * @size: the command size
 * Return: 0 - success, negative - failure
 */
int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
                        enum hinic_node_id dest, void *cmd, u16 size)
{
        /* Verify the chain type */
        return api_cmd(chain, dest, cmd, size, NULL, 0);
}

/**
 * api_cmd_hw_restart - restart the chain in the HW
 * @chain: the API CMD specific chain to restart
 */
static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        unsigned long end;
        u32 reg_addr, val;
        int err;

        /* Read Modify Write */
        reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
        val = hinic_hwif_read_reg(hwif, reg_addr);

        val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
        val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);

        hinic_hwif_write_reg(hwif, reg_addr, val);

        end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
        err = -ETIMEDOUT;
        do {
                val = hinic_hwif_read_reg(hwif, reg_addr);

                if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
                        err = 0;
                        break;
                }

                rte_delay_ms(1);
        } while (time_before(jiffies, end));

        return err;
}

/**
 * api_cmd_ctrl_init - set the control register of a chain
 * @chain: the API CMD specific chain to set control register for
 */
static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 reg_addr, ctrl;
        u32 cell_size;

        /* Read Modify Write */
        reg_addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);

        cell_size = (u32)ilog2(chain->cell_size >>
                               API_CMD_CHAIN_CELL_SIZE_SHIFT);

        ctrl = hinic_hwif_read_reg(hwif, reg_addr);

        ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
                HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);

        ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(0, AEQE_EN) |
                HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);

        hinic_hwif_write_reg(hwif, reg_addr, ctrl);
}

/**
 * api_cmd_set_status_addr - set the status address of a chain in the HW
 * @chain: the API CMD specific chain to set status address for
 */
static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 addr, val;

        addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
        val = upper_32_bits(chain->wb_status_paddr);
        hinic_hwif_write_reg(hwif, addr, val);

        addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
        val = lower_32_bits(chain->wb_status_paddr);
        hinic_hwif_write_reg(hwif, addr, val);
}

/**
 * api_cmd_set_num_cells - set the number cells of a chain in the HW
 * @chain: the API CMD specific chain to set the number of cells for
 */
static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 addr, val;

        addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
        val  = chain->num_cells;
        hinic_hwif_write_reg(hwif, addr, val);
}

/**
 * api_cmd_head_init - set the head cell of a chain in the HW
 * @chain: the API CMD specific chain to set the head for
 */
static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 addr, val;

        addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
        val = upper_32_bits(chain->head_cell_paddr);
        hinic_hwif_write_reg(hwif, addr, val);

        addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
        val = lower_32_bits(chain->head_cell_paddr);
        hinic_hwif_write_reg(hwif, addr, val);
}

/**
 * wait_for_ready_chain - wait for the chain to be ready
 * @chain: the API CMD specific chain to wait for
 * Return: 0 - success, negative - failure
 */
static int wait_for_ready_chain(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        unsigned long end;
        u32 addr, val;
        u32 hw_cons_idx;
        int err;

        end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);

        addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
        err = -ETIMEDOUT;
        do {
                val = hinic_hwif_read_reg(hwif, addr);
                hw_cons_idx = HINIC_API_CMD_STATUS_GET(val, CONS_IDX);

                /* Wait for HW cons idx to be updated */
                if (hw_cons_idx == chain->cons_idx) {
                        err = 0;
                        break;
                }

                rte_delay_ms(1);
        } while (time_before(jiffies, end));

        return err;
}

/**
 * api_cmd_chain_hw_clean - clean the HW
 * @chain: the API CMD specific chain
 */
static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
{
        struct hinic_hwif *hwif = chain->hwdev->hwif;
        u32 addr, ctrl;

        addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);

        ctrl = hinic_hwif_read_reg(hwif, addr);
        ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_EN) &
               HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)    &
               HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)    &
               HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
               HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);

        hinic_hwif_write_reg(hwif, addr, ctrl);
}

/**
 * api_cmd_chain_hw_init - initialize the chain in the HW
 *(initialize API command csr)
 * @chain: the API CMD specific chain to initialize in HW
 * Return: 0 - success, negative - failure
 */
static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
{
        api_cmd_chain_hw_clean(chain);

        api_cmd_set_status_addr(chain);

        if (api_cmd_hw_restart(chain)) {
                PMD_DRV_LOG(ERR, "Restart api_cmd_hw failed");
                return -EBUSY;
        }

        api_cmd_ctrl_init(chain);
        api_cmd_set_num_cells(chain);
        api_cmd_head_init(chain);

        return wait_for_ready_chain(chain);
}

/**
 * free_cmd_buf - free the dma buffer of API CMD command
 * @chain: the API CMD specific chain of the cmd
 * @cell_idx: the cell index of the cmd
 */
static void free_cmd_buf(struct hinic_api_cmd_chain *chain, u32 cell_idx)
{
        struct hinic_api_cmd_cell_ctxt *cell_ctxt;
        void *dev = chain->hwdev;

        cell_ctxt = &chain->cell_ctxt[cell_idx];

        dma_free_coherent(dev, (API_CMD_BUF_SIZE + API_PAYLOAD_ALIGN_SIZE),
                          cell_ctxt->api_cmd_vaddr_free,
                          cell_ctxt->api_cmd_paddr_free);
}

/**
 * alloc_cmd_buf - allocate a dma buffer for API CMD command
 * @chain: the API CMD specific chain for the cmd
 * @cell: the cell in the HW for the cmd
 * @cell_idx: the index of the cell
 * Return: 0 - success, negative - failure
 */
static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
                         struct hinic_api_cmd_cell *cell, u32 cell_idx)
{
        void *dev = chain->hwdev;
        struct hinic_api_cmd_cell_ctxt *cell_ctxt;
        dma_addr_t cmd_paddr = 0;
        void *cmd_vaddr;
        void *cmd_vaddr_alloc;
        int err = 0;

        cmd_vaddr_alloc = dma_zalloc_coherent(dev, (API_CMD_BUF_SIZE +
                                              API_PAYLOAD_ALIGN_SIZE),
                                              &cmd_paddr, SOCKET_ID_ANY);
        if (!cmd_vaddr_alloc) {
                PMD_DRV_LOG(ERR, "Allocate API CMD dma memory failed");
                return -ENOMEM;
        }

        cell_ctxt = &chain->cell_ctxt[cell_idx];

        cell_ctxt->api_cmd_paddr_free = cmd_paddr;
        cell_ctxt->api_cmd_vaddr_free = cmd_vaddr_alloc;
        cmd_vaddr = PTR_ALIGN(cmd_vaddr_alloc, API_PAYLOAD_ALIGN_SIZE);
        cmd_paddr = cmd_paddr + ((u64)cmd_vaddr - (u64)cmd_vaddr_alloc);

        cell_ctxt->api_cmd_vaddr = cmd_vaddr;
        cell_ctxt->api_cmd_paddr = cmd_paddr;

        /* set the cmd DMA address in the cell */
        switch (chain->chain_type) {
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
                cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
                break;
        default:
                PMD_DRV_LOG(ERR, "Unknown API CMD chain type");
                free_cmd_buf(chain, cell_idx);
                err = -EINVAL;
                break;
        }

        return err;
}

/**
 * api_cmd_create_cell - create API CMD cell of specific chain
 * @chain: the API CMD specific chain to create its cell
 * @cell_idx: the cell index to create
 * @pre_node: previous cell
 * @node_vaddr: the virt addr of the cell
 * Return: 0 - success, negative - failure
 */
static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
                               u32 cell_idx,
                               struct hinic_api_cmd_cell *pre_node,
                               struct hinic_api_cmd_cell **node_vaddr)
{
        void *dev = chain->hwdev;
        struct hinic_api_cmd_cell_ctxt *cell_ctxt;
        struct hinic_api_cmd_cell *node;
        dma_addr_t node_paddr = 0;
        void *node_vaddr_alloc;
        int err = 0;

        node_vaddr_alloc = dma_zalloc_coherent(dev, (chain->cell_size +
                                               API_CMD_NODE_ALIGN_SIZE),
                                               &node_paddr, SOCKET_ID_ANY);
        if (!node_vaddr_alloc) {
                PMD_DRV_LOG(ERR, "Allocate dma API CMD cell failed");
                return -ENOMEM;
        }

        cell_ctxt = &chain->cell_ctxt[cell_idx];

        cell_ctxt->cell_vaddr_free = node_vaddr_alloc;
        cell_ctxt->cell_paddr_free = node_paddr;
        node = (struct hinic_api_cmd_cell *)PTR_ALIGN(node_vaddr_alloc,
                API_CMD_NODE_ALIGN_SIZE);
        node_paddr = node_paddr + ((u64)node - (u64)node_vaddr_alloc);

        node->read.hw_wb_resp_paddr = 0;

        cell_ctxt->cell_vaddr = node;
        cell_ctxt->cell_paddr = node_paddr;

        if (!pre_node) {
                chain->head_node = node;
                chain->head_cell_paddr = node_paddr;
        } else {
                /* The data in the HW should be in Big Endian Format */
                pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
        }

        /* Driver software should make sure that there is an empty
         * API command cell at the end the chain
         */
        node->next_cell_paddr = 0;

        switch (chain->chain_type) {
        case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
        case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                err = alloc_cmd_buf(chain, node, cell_idx);
                if (err) {
                        PMD_DRV_LOG(ERR, "Allocate cmd buffer failed");
                        goto alloc_cmd_buf_err;
                }
                break;
        default:
                PMD_DRV_LOG(ERR, "Unsupported API CMD chain type");
                err = -EINVAL;
                goto alloc_cmd_buf_err;
        }

        *node_vaddr = node;

        return 0;

alloc_cmd_buf_err:
        dma_free_coherent(dev, (chain->cell_size + API_CMD_NODE_ALIGN_SIZE),
                          node_vaddr_alloc, cell_ctxt->cell_paddr_free);

        return err;
}

/**
 * api_cmd_destroy_cell - destroy API CMD cell of specific chain
 * @chain: the API CMD specific chain to destroy its cell
 * @cell_idx: the cell to destroy
 */
static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
                                 u32 cell_idx)
{
        void *dev = chain->hwdev;
        struct hinic_api_cmd_cell_ctxt *cell_ctxt;
        struct hinic_api_cmd_cell *node;
        dma_addr_t node_paddr;

        cell_ctxt = &chain->cell_ctxt[cell_idx];

        node = (struct hinic_api_cmd_cell *)(cell_ctxt->cell_vaddr_free);
        node_paddr = cell_ctxt->cell_paddr_free;

        if (cell_ctxt->api_cmd_vaddr) {
                switch (chain->chain_type) {
                case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
                case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
                        free_cmd_buf(chain, cell_idx);
                        break;
                default:
                        break;
                }

                dma_free_coherent(dev, (chain->cell_size +
                                  API_CMD_NODE_ALIGN_SIZE),
                                  node, node_paddr);
        }
}

/**
 * api_cmd_destroy_cells - destroy API CMD cells of specific chain
 * @chain: the API CMD specific chain to destroy its cells
 * @num_cells: number of cells to destroy
 */
static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
                                         u32 num_cells)
{
        u32 cell_idx;

        for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
                api_cmd_destroy_cell(chain, cell_idx);
}

/**
 * api_cmd_create_cells - create API CMD cells for specific chain
 * @chain: the API CMD specific chain
 * Return: 0 - success, negative - failure
 */
static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
{
        struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
        u32 cell_idx;
        int err;

        for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
                err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
                if (err) {
                        PMD_DRV_LOG(ERR, "Create API CMD cell failed");
                        goto create_cell_err;
                }

                pre_node = node;
        }

        if (!node) {
                err = -EFAULT;
                goto create_cell_err;
        }

        /* set the Final node to point on the start */
        node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);

        /* set the current node to be the head */
        chain->curr_node = chain->head_node;
        return 0;

create_cell_err:
        api_cmd_destroy_cells(chain, cell_idx);
        return err;
}

/**
 * api_chain_init - initialize API CMD specific chain
 * @chain: the API CMD specific chain to initialize
 * @attr: attributes to set in the chain
 * Return: 0 - success, negative - failure
 */
static int api_chain_init(struct hinic_api_cmd_chain *chain,
                          struct hinic_api_cmd_chain_attr *attr)
{
        void *dev = chain->hwdev;
        size_t cell_ctxt_size;
        int err;

        chain->chain_type  = attr->chain_type;
        chain->num_cells = attr->num_cells;
        chain->cell_size = attr->cell_size;
        chain->rsp_size = attr->rsp_size;

        chain->prod_idx  = 0;
        chain->cons_idx  = 0;

        spin_lock_init(&chain->async_lock);

        cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
        chain->cell_ctxt = kzalloc(cell_ctxt_size, GFP_KERNEL);
        if (!chain->cell_ctxt) {
                PMD_DRV_LOG(ERR, "Allocate cell contexts for a chain failed");
                err = -ENOMEM;
                goto alloc_cell_ctxt_err;
        }

        chain->wb_status = (struct hinic_api_cmd_status *)
                           dma_zalloc_coherent(dev, sizeof(*chain->wb_status),
                                &chain->wb_status_paddr, SOCKET_ID_ANY);
        if (!chain->wb_status) {
                PMD_DRV_LOG(ERR, "Allocate DMA wb status failed");
                err = -ENOMEM;
                goto alloc_wb_status_err;
        }

        return 0;

alloc_wb_status_err:
        kfree(chain->cell_ctxt);

alloc_cell_ctxt_err:

        return err;
}

/**
 * api_chain_free - free API CMD specific chain
 * @chain: the API CMD specific chain to free
 */
static void api_chain_free(struct hinic_api_cmd_chain *chain)
{
        void *dev = chain->hwdev;

        dma_free_coherent(dev, sizeof(*chain->wb_status),
                          chain->wb_status, chain->wb_status_paddr);
        kfree(chain->cell_ctxt);
}

/**
 * api_cmd_create_chain - create API CMD specific chain
 * @cmd_chain: the API CMD specific chain to create
 * @attr: attributes to set in the chain
 * Return: 0 - success, negative - failure
 */
static int api_cmd_create_chain(struct hinic_api_cmd_chain **cmd_chain,
                                struct hinic_api_cmd_chain_attr *attr)
{
        struct hinic_hwdev *hwdev = attr->hwdev;
        struct hinic_api_cmd_chain *chain;
        int err;

        if (attr->num_cells & (attr->num_cells - 1)) {
                PMD_DRV_LOG(ERR, "Invalid number of cells, must be power of 2");
                return -EINVAL;
        }

        chain = kzalloc(sizeof(*chain), GFP_KERNEL);
        if (!chain) {
                PMD_DRV_LOG(ERR, "Allocate memory for the chain failed");
                return -ENOMEM;
        }

        chain->hwdev = hwdev;

        err = api_chain_init(chain, attr);
        if (err) {
                PMD_DRV_LOG(ERR, "Initialize chain failed");
                goto chain_init_err;
        }

        err = api_cmd_create_cells(chain);
        if (err) {
                PMD_DRV_LOG(ERR, "Create cells for API CMD chain failed");
                goto create_cells_err;
        }

        err = api_cmd_chain_hw_init(chain);
        if (err) {
                PMD_DRV_LOG(ERR, "Initialize chain hw info failed");
                goto chain_hw_init_err;
        }

        *cmd_chain = chain;
        return 0;

chain_hw_init_err:
        api_cmd_destroy_cells(chain, chain->num_cells);

create_cells_err:
        api_chain_free(chain);

chain_init_err:
        kfree(chain);
        return err;
}

/**
 * api_cmd_destroy_chain - destroy API CMD specific chain
 * @chain: the API CMD specific chain to destroy
 */
static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
{
        api_cmd_destroy_cells(chain, chain->num_cells);
        api_chain_free(chain);
        kfree(chain);
}

/**
 * hinic_api_cmd_init - Initialize all the API CMD chains
 * @hwdev: the hardware interface of a pci function device
 * @chain: the API CMD chains that will be initialized
 * Return: 0 - success, negative - failure
 */
int hinic_api_cmd_init(struct hinic_hwdev *hwdev,
                       struct hinic_api_cmd_chain **chain)
{
        struct hinic_api_cmd_chain_attr attr;

        enum hinic_api_cmd_chain_type chain_type, i;
        int err;

        attr.hwdev = hwdev;
        attr.num_cells  = API_CHAIN_NUM_CELLS;
        attr.cell_size  = API_CHAIN_CELL_SIZE;
        attr.rsp_size   = API_CHAIN_RSP_DATA_SIZE;

        chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
        for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
                attr.chain_type = chain_type;
                err = api_cmd_create_chain(&chain[chain_type], &attr);
                if (err) {
                        PMD_DRV_LOG(ERR, "Create chain %d failed",
                                chain_type);
                        goto create_chain_err;
                }
        }

        return 0;

create_chain_err:
        i = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
        for (; i < chain_type; i++)
                api_cmd_destroy_chain(chain[i]);

        return err;
}

/**
 * hinic_api_cmd_free - free the API CMD chains
 * @chain: the API CMD chains that will be freed
 */
void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
{
        enum hinic_api_cmd_chain_type chain_type;

        chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
        for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++)
                api_cmd_destroy_chain(chain[chain_type]);
}