/*
 * Copyright (c) 2016-2017 Hisilicon Limited.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/acpi.h>
#include <linux/etherdevice.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <net/addrconf.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_umem.h>
#include <rdma/uverbs_ioctl.h>

#include "hnae3.h"
#include "hns_roce_common.h"
#include "hns_roce_device.h"
#include "hns_roce_cmd.h"
#include "hns_roce_hem.h"
#include "hns_roce_hw_v2.h"

static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
                            struct ib_sge *sg)
{
        dseg->lkey = cpu_to_le32(sg->lkey);
        dseg->addr = cpu_to_le64(sg->addr);
        dseg->len  = cpu_to_le32(sg->length);
}

/*
 * mapped-value = 1 + real-value
 * The hns wr opcode real value is start from 0, In order to distinguish between
 * initialized and uninitialized map values, we plus 1 to the actual value when
 * defining the mapping, so that the validity can be identified by checking the
 * mapped value is greater than 0.
 */
#define HR_OPC_MAP(ib_key, hr_key) \
                [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key

static const u32 hns_roce_op_code[] = {
        HR_OPC_MAP(RDMA_WRITE,                  RDMA_WRITE),
        HR_OPC_MAP(RDMA_WRITE_WITH_IMM,         RDMA_WRITE_WITH_IMM),
        HR_OPC_MAP(SEND,                        SEND),
        HR_OPC_MAP(SEND_WITH_IMM,               SEND_WITH_IMM),
        HR_OPC_MAP(RDMA_READ,                   RDMA_READ),
        HR_OPC_MAP(ATOMIC_CMP_AND_SWP,          ATOM_CMP_AND_SWAP),
        HR_OPC_MAP(ATOMIC_FETCH_AND_ADD,        ATOM_FETCH_AND_ADD),
        HR_OPC_MAP(SEND_WITH_INV,               SEND_WITH_INV),
        HR_OPC_MAP(LOCAL_INV,                   LOCAL_INV),
        HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP,   ATOM_MSK_CMP_AND_SWAP),
        HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
        HR_OPC_MAP(REG_MR,                      FAST_REG_PMR),
};

static u32 to_hr_opcode(u32 ib_opcode)
{
        if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
                return HNS_ROCE_V2_WQE_OP_MASK;

        return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
                                             HNS_ROCE_V2_WQE_OP_MASK;
}

static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
                         const struct ib_reg_wr *wr)
{
        struct hns_roce_wqe_frmr_seg *fseg =
                (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
        struct hns_roce_mr *mr = to_hr_mr(wr->mr);
        u64 pbl_ba;

        /* use ib_access_flags */
        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
                     wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
                     wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S,
                     wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S,
                     wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S,
                     wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);

        /* Data structure reuse may lead to confusion */
        pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
        rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
        rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));

        rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
        rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
        rc_sq_wqe->rkey = cpu_to_le32(wr->key);
        rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);

        fseg->pbl_size = cpu_to_le32(mr->npages);
        roce_set_field(fseg->mode_buf_pg_sz,
                       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
                       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
                       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
        roce_set_bit(fseg->mode_buf_pg_sz,
                     V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
}

static void set_atomic_seg(const struct ib_send_wr *wr,
                           struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
                           unsigned int valid_num_sge)
{
        struct hns_roce_v2_wqe_data_seg *dseg =
                (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
        struct hns_roce_wqe_atomic_seg *aseg =
                (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);

        set_data_seg_v2(dseg, wr->sg_list);

        if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
                aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
                aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
        } else {
                aseg->fetchadd_swap_data =
                        cpu_to_le64(atomic_wr(wr)->compare_add);
                aseg->cmp_data = 0;
        }

        roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
                       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
}

static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
                           unsigned int *sge_ind, unsigned int valid_num_sge)
{
        struct hns_roce_v2_wqe_data_seg *dseg;
        unsigned int cnt = valid_num_sge;
        struct ib_sge *sge = wr->sg_list;
        unsigned int idx = *sge_ind;

        if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
                cnt -= HNS_ROCE_SGE_IN_WQE;
                sge += HNS_ROCE_SGE_IN_WQE;
        }

        while (cnt > 0) {
                dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
                set_data_seg_v2(dseg, sge);
                idx++;
                sge++;
                cnt--;
        }

        *sge_ind = idx;
}

static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
                             struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
                             unsigned int *sge_ind,
                             unsigned int valid_num_sge)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
        struct hns_roce_v2_wqe_data_seg *dseg =
                (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
        struct ib_device *ibdev = &hr_dev->ib_dev;
        struct hns_roce_qp *qp = to_hr_qp(ibqp);
        void *wqe = dseg;
        int j = 0;
        int i;

        if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) {
                if (unlikely(le32_to_cpu(rc_sq_wqe->msg_len) >
                             hr_dev->caps.max_sq_inline)) {
                        ibdev_err(ibdev, "inline len(1-%d)=%d, illegal",
                                  rc_sq_wqe->msg_len,
                                  hr_dev->caps.max_sq_inline);
                        return -EINVAL;
                }

                if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
                        ibdev_err(ibdev, "Not support inline data!\n");
                        return -EINVAL;
                }

                for (i = 0; i < wr->num_sge; i++) {
                        memcpy(wqe, ((void *)wr->sg_list[i].addr),
                               wr->sg_list[i].length);
                        wqe += wr->sg_list[i].length;
                }

                roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
                             1);
        } else {
                if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
                        for (i = 0; i < wr->num_sge; i++) {
                                if (likely(wr->sg_list[i].length)) {
                                        set_data_seg_v2(dseg, wr->sg_list + i);
                                        dseg++;
                                }
                        }
                } else {
                        roce_set_field(rc_sq_wqe->byte_20,
                                     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
                                     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
                                     (*sge_ind) & (qp->sge.sge_cnt - 1));

                        for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE;
                             i++) {
                                if (likely(wr->sg_list[i].length)) {
                                        set_data_seg_v2(dseg, wr->sg_list + i);
                                        dseg++;
                                        j++;
                                }
                        }

                        set_extend_sge(qp, wr, sge_ind, valid_num_sge);
                }

                roce_set_field(rc_sq_wqe->byte_16,
                               V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
                               V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
        }

        return 0;
}

static int check_send_valid(struct hns_roce_dev *hr_dev,
                            struct hns_roce_qp *hr_qp)
{
        struct ib_device *ibdev = &hr_dev->ib_dev;
        struct ib_qp *ibqp = &hr_qp->ibqp;

        if (unlikely(ibqp->qp_type != IB_QPT_RC &&
                     ibqp->qp_type != IB_QPT_GSI &&
                     ibqp->qp_type != IB_QPT_UD)) {
                ibdev_err(ibdev, "Not supported QP(0x%x)type!\n",
                          ibqp->qp_type);
                return -EOPNOTSUPP;
        } else if (unlikely(hr_qp->state == IB_QPS_RESET ||
                   hr_qp->state == IB_QPS_INIT ||
                   hr_qp->state == IB_QPS_RTR)) {
                ibdev_err(ibdev, "failed to post WQE, QP state %d!\n",
                          hr_qp->state);
                return -EINVAL;
        } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
                ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",
                          hr_dev->state);
                return -EIO;
        }

        return 0;
}

static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
                                    unsigned int *sge_len)
{
        unsigned int valid_num = 0;
        unsigned int len = 0;
        int i;

        for (i = 0; i < wr->num_sge; i++) {
                if (likely(wr->sg_list[i].length)) {
                        len += wr->sg_list[i].length;
                        valid_num++;
                }
        }

        *sge_len = len;
        return valid_num;
}

static inline int set_ud_wqe(struct hns_roce_qp *qp,
                             const struct ib_send_wr *wr,
                             void *wqe, unsigned int *sge_idx,
                             unsigned int owner_bit)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
        struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
        struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
        unsigned int curr_idx = *sge_idx;
        int valid_num_sge;
        u32 msg_len = 0;
        bool loopback;
        u8 *smac;

        valid_num_sge = calc_wr_sge_num(wr, &msg_len);
        memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));

        roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
                       V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
        roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
                       V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
        roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
                       V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
        roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
                       V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
        roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
                       V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]);
        roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
                       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);

        /* MAC loopback */
        smac = (u8 *)hr_dev->dev_addr[qp->port];
        loopback = ether_addr_equal_unaligned(ah->av.mac, smac) ? 1 : 0;

        roce_set_bit(ud_sq_wqe->byte_40,
                     V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);

        roce_set_field(ud_sq_wqe->byte_4,
                       V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
                       V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
                       HNS_ROCE_V2_WQE_OP_SEND);

        ud_sq_wqe->msg_len = cpu_to_le32(msg_len);

        switch (wr->opcode) {
        case IB_WR_SEND_WITH_IMM:
        case IB_WR_RDMA_WRITE_WITH_IMM:
                ud_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
                break;
        default:
                ud_sq_wqe->immtdata = 0;
                break;
        }

        /* Set sig attr */
        roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
                     (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);

        /* Set se attr */
        roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
                     (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);

        roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
                     owner_bit);

        roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
                       V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);

        roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
                       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);

        roce_set_field(ud_sq_wqe->byte_20,
                       V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
                       V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
                       curr_idx & (qp->sge.sge_cnt - 1));

        roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
                       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
        ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
                          qp->qkey : ud_wr(wr)->remote_qkey);
        roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
                       V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);

        roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
                       V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
        roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
                       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
        roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
                       V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
        roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
                       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
        roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
                       V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
        roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
                       V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);

        roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
                     ah->av.vlan_en ? 1 : 0);
        roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
                       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);

        memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);

        set_extend_sge(qp, wr, &curr_idx, valid_num_sge);

        *sge_idx = curr_idx;

        return 0;
}

static inline int set_rc_wqe(struct hns_roce_qp *qp,
                             const struct ib_send_wr *wr,
                             void *wqe, unsigned int *sge_idx,
                             unsigned int owner_bit)
{
        struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
        unsigned int curr_idx = *sge_idx;
        unsigned int valid_num_sge;
        u32 msg_len = 0;
        int ret = 0;

        valid_num_sge = calc_wr_sge_num(wr, &msg_len);
        memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));

        rc_sq_wqe->msg_len = cpu_to_le32(msg_len);

        switch (wr->opcode) {
        case IB_WR_SEND_WITH_IMM:
        case IB_WR_RDMA_WRITE_WITH_IMM:
                rc_sq_wqe->immtdata = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
                break;
        case IB_WR_SEND_WITH_INV:
                rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
                break;
        default:
                rc_sq_wqe->immtdata = 0;
                break;
        }

        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
                     (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);

        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
                     (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);

        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
                     (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);

        roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
                     owner_bit);

        switch (wr->opcode) {
        case IB_WR_RDMA_READ:
        case IB_WR_RDMA_WRITE:
        case IB_WR_RDMA_WRITE_WITH_IMM:
                rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
                rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
                break;
        case IB_WR_LOCAL_INV:
                roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
                rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
                break;
        case IB_WR_REG_MR:
                set_frmr_seg(rc_sq_wqe, reg_wr(wr));
                break;
        case IB_WR_ATOMIC_CMP_AND_SWP:
        case IB_WR_ATOMIC_FETCH_AND_ADD:
                rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
                rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
                break;
        default:
                break;
        }

        roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
                       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
                       to_hr_opcode(wr->opcode));

        if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
            wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
                set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
        else if (wr->opcode != IB_WR_REG_MR)
                ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
                                        &curr_idx, valid_num_sge);

        *sge_idx = curr_idx;

        return ret;
}

static inline void update_sq_db(struct hns_roce_dev *hr_dev,
                                struct hns_roce_qp *qp)
{
        /*
         * Hip08 hardware cannot flush the WQEs in SQ if the QP state
         * gets into errored mode. Hence, as a workaround to this
         * hardware limitation, driver needs to assist in flushing. But
         * the flushing operation uses mailbox to convey the QP state to
         * the hardware and which can sleep due to the mutex protection
         * around the mailbox calls. Hence, use the deferred flush for
         * now.
         */
        if (qp->state == IB_QPS_ERR) {
                if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
                        init_flush_work(hr_dev, qp);
        } else {
                struct hns_roce_v2_db sq_db = {};

                roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
                               V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
                roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
                               V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
                roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
                               V2_DB_PARAMETER_IDX_S, qp->sq.head);
                roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
                               V2_DB_PARAMETER_SL_S, qp->sl);

                hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
        }
}

static int hns_roce_v2_post_send(struct ib_qp *ibqp,
                                 const struct ib_send_wr *wr,
                                 const struct ib_send_wr **bad_wr)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
        struct ib_device *ibdev = &hr_dev->ib_dev;
        struct hns_roce_qp *qp = to_hr_qp(ibqp);
        unsigned long flags = 0;
        unsigned int owner_bit;
        unsigned int sge_idx;
        unsigned int wqe_idx;
        void *wqe = NULL;
        int nreq;
        int ret;

        spin_lock_irqsave(&qp->sq.lock, flags);

        ret = check_send_valid(hr_dev, qp);
        if (unlikely(ret)) {
                *bad_wr = wr;
                nreq = 0;
                goto out;
        }

        sge_idx = qp->next_sge;

        for (nreq = 0; wr; ++nreq, wr = wr->next) {
                if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
                        ret = -ENOMEM;
                        *bad_wr = wr;
                        goto out;
                }

                wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);

                if (unlikely(wr->num_sge > qp->sq.max_gs)) {
                        ibdev_err(ibdev, "num_sge=%d > qp->sq.max_gs=%d\n",
                                  wr->num_sge, qp->sq.max_gs);
                        ret = -EINVAL;
                        *bad_wr = wr;
                        goto out;
                }

                wqe = hns_roce_get_send_wqe(qp, wqe_idx);
                qp->sq.wrid[wqe_idx] = wr->wr_id;
                owner_bit =
                       ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);

                /* Corresponding to the QP type, wqe process separately */
                if (ibqp->qp_type == IB_QPT_GSI)
                        ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
                else if (ibqp->qp_type == IB_QPT_RC)
                        ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);

                if (unlikely(ret)) {
                        *bad_wr = wr;
                        goto out;
                }
        }

out:
        if (likely(nreq)) {
                qp->sq.head += nreq;
                qp->next_sge = sge_idx;
                /* Memory barrier */
                wmb();
                update_sq_db(hr_dev, qp);
        }

        spin_unlock_irqrestore(&qp->sq.lock, flags);

        return ret;
}

static int check_recv_valid(struct hns_roce_dev *hr_dev,
                            struct hns_roce_qp *hr_qp)
{
        if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
                return -EIO;
        else if (hr_qp->state == IB_QPS_RESET)
                return -EINVAL;

        return 0;
}

static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
                                 const struct ib_recv_wr *wr,
                                 const struct ib_recv_wr **bad_wr)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
        struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
        struct ib_device *ibdev = &hr_dev->ib_dev;
        struct hns_roce_v2_wqe_data_seg *dseg;
        struct hns_roce_rinl_sge *sge_list;
        unsigned long flags;
        void *wqe = NULL;
        u32 wqe_idx;
        int nreq;
        int ret;
        int i;

        spin_lock_irqsave(&hr_qp->rq.lock, flags);

        ret = check_recv_valid(hr_dev, hr_qp);
        if (unlikely(ret)) {
                *bad_wr = wr;
                nreq = 0;
                goto out;
        }

        for (nreq = 0; wr; ++nreq, wr = wr->next) {
                if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
                                                  hr_qp->ibqp.recv_cq))) {
                        ret = -ENOMEM;
                        *bad_wr = wr;
                        goto out;
                }

                wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);

                if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
                        ibdev_err(ibdev, "rq:num_sge=%d >= qp->sq.max_gs=%d\n",
                                  wr->num_sge, hr_qp->rq.max_gs);
                        ret = -EINVAL;
                        *bad_wr = wr;
                        goto out;
                }

                wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
                dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
                for (i = 0; i < wr->num_sge; i++) {
                        if (!wr->sg_list[i].length)
                                continue;
                        set_data_seg_v2(dseg, wr->sg_list + i);
                        dseg++;
                }

                if (wr->num_sge < hr_qp->rq.max_gs) {
                        dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
                        dseg->addr = 0;
                }

                /* rq support inline data */
                if (hr_qp->rq_inl_buf.wqe_cnt) {
                        sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
                        hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt =
                                                               (u32)wr->num_sge;
                        for (i = 0; i < wr->num_sge; i++) {
                                sge_list[i].addr =
                                               (void *)(u64)wr->sg_list[i].addr;
                                sge_list[i].len = wr->sg_list[i].length;
                        }
                }

                hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
        }

out:
        if (likely(nreq)) {
                hr_qp->rq.head += nreq;
                /* Memory barrier */
                wmb();

                /*
                 * Hip08 hardware cannot flush the WQEs in RQ if the QP state
                 * gets into errored mode. Hence, as a workaround to this
                 * hardware limitation, driver needs to assist in flushing. But
                 * the flushing operation uses mailbox to convey the QP state to
                 * the hardware and which can sleep due to the mutex protection
                 * around the mailbox calls. Hence, use the deferred flush for
                 * now.
                 */
                if (hr_qp->state == IB_QPS_ERR) {
                        if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG,
                                              &hr_qp->flush_flag))
                                init_flush_work(hr_dev, hr_qp);
                } else {
                        *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
                }
        }
        spin_unlock_irqrestore(&hr_qp->rq.lock, flags);

        return ret;
}

static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
{
        return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
}

static void *get_idx_buf(struct hns_roce_idx_que *idx_que, int n)
{
        return hns_roce_buf_offset(idx_que->mtr.kmem,
                                   n << idx_que->entry_shift);
}

static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
{
        /* always called with interrupts disabled. */
        spin_lock(&srq->lock);

        bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
        srq->tail++;

        spin_unlock(&srq->lock);
}

static int find_empty_entry(struct hns_roce_idx_que *idx_que,
                            unsigned long size)
{
        int wqe_idx;

        if (unlikely(bitmap_full(idx_que->bitmap, size)))
                return -ENOSPC;

        wqe_idx = find_first_zero_bit(idx_que->bitmap, size);

        bitmap_set(idx_que->bitmap, wqe_idx, 1);

        return wqe_idx;
}

static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
                                     const struct ib_recv_wr *wr,
                                     const struct ib_recv_wr **bad_wr)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
        struct hns_roce_srq *srq = to_hr_srq(ibsrq);
        struct hns_roce_v2_wqe_data_seg *dseg;
        struct hns_roce_v2_db srq_db;
        unsigned long flags;
        __le32 *srq_idx;
        int ret = 0;
        int wqe_idx;
        void *wqe;
        int nreq;
        int ind;
        int i;

        spin_lock_irqsave(&srq->lock, flags);

        ind = srq->head & (srq->wqe_cnt - 1);

        for (nreq = 0; wr; ++nreq, wr = wr->next) {
                if (unlikely(wr->num_sge >= srq->max_gs)) {
                        ret = -EINVAL;
                        *bad_wr = wr;
                        break;
                }

                if (unlikely(srq->head == srq->tail)) {
                        ret = -ENOMEM;
                        *bad_wr = wr;
                        break;
                }

                wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
                if (unlikely(wqe_idx < 0)) {
                        ret = -ENOMEM;
                        *bad_wr = wr;
                        break;
                }

                wqe = get_srq_wqe(srq, wqe_idx);
                dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;

                for (i = 0; i < wr->num_sge; ++i) {
                        dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
                        dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
                        dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
                }

                if (wr->num_sge < srq->max_gs) {
                        dseg[i].len = 0;
                        dseg[i].lkey = cpu_to_le32(0x100);
                        dseg[i].addr = 0;
                }

                srq_idx = get_idx_buf(&srq->idx_que, ind);
                *srq_idx = cpu_to_le32(wqe_idx);

                srq->wrid[wqe_idx] = wr->wr_id;
                ind = (ind + 1) & (srq->wqe_cnt - 1);
        }

        if (likely(nreq)) {
                srq->head += nreq;

                /*
                 * Make sure that descriptors are written before
                 * doorbell record.
                 */
                wmb();

                srq_db.byte_4 =
                        cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
                                    (srq->srqn & V2_DB_BYTE_4_TAG_M));
                srq_db.parameter =
                        cpu_to_le32(srq->head & V2_DB_PARAMETER_IDX_M);

                hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
        }

        spin_unlock_irqrestore(&srq->lock, flags);

        return ret;
}

static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
                                      unsigned long instance_stage,
                                      unsigned long reset_stage)
{
        /* When hardware reset has been completed once or more, we should stop
         * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
         * function, we should exit with error. If now at HNAE3_INIT_CLIENT
         * stage of soft reset process, we should exit with error, and then
         * HNAE3_INIT_CLIENT related process can rollback the operation like
         * notifing hardware to free resources, HNAE3_INIT_CLIENT related
         * process will exit with error to notify NIC driver to reschedule soft
         * reset process once again.
         */
        hr_dev->is_reset = true;
        hr_dev->dis_db = true;

        if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
            instance_stage == HNS_ROCE_STATE_INIT)
                return CMD_RST_PRC_EBUSY;

        return CMD_RST_PRC_SUCCESS;
}

static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
                                        unsigned long instance_stage,
                                        unsigned long reset_stage)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hnae3_handle *handle = priv->handle;
        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;

        /* When hardware reset is detected, we should stop sending mailbox&cmq&
         * doorbell to hardware. If now in .init_instance() function, we should
         * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
         * process, we should exit with error, and then HNAE3_INIT_CLIENT
         * related process can rollback the operation like notifing hardware to
         * free resources, HNAE3_INIT_CLIENT related process will exit with
         * error to notify NIC driver to reschedule soft reset process once
         * again.
         */
        hr_dev->dis_db = true;
        if (!ops->get_hw_reset_stat(handle))
                hr_dev->is_reset = true;

        if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
            instance_stage == HNS_ROCE_STATE_INIT)
                return CMD_RST_PRC_EBUSY;

        return CMD_RST_PRC_SUCCESS;
}

static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hnae3_handle *handle = priv->handle;
        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;

        /* When software reset is detected at .init_instance() function, we
         * should stop sending mailbox&cmq&doorbell to hardware, and exit
         * with error.
         */
        hr_dev->dis_db = true;
        if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
                hr_dev->is_reset = true;

        return CMD_RST_PRC_EBUSY;
}

static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hnae3_handle *handle = priv->handle;
        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
        unsigned long instance_stage;   /* the current instance stage */
        unsigned long reset_stage;      /* the current reset stage */
        unsigned long reset_cnt;
        bool sw_resetting;
        bool hw_resetting;

        if (hr_dev->is_reset)
                return CMD_RST_PRC_SUCCESS;

        /* Get information about reset from NIC driver or RoCE driver itself,
         * the meaning of the following variables from NIC driver are described
         * as below:
         * reset_cnt -- The count value of completed hardware reset.
         * hw_resetting -- Whether hardware device is resetting now.
         * sw_resetting -- Whether NIC's software reset process is running now.
         */
        instance_stage = handle->rinfo.instance_state;
        reset_stage = handle->rinfo.reset_state;
        reset_cnt = ops->ae_dev_reset_cnt(handle);
        hw_resetting = ops->get_cmdq_stat(handle);
        sw_resetting = ops->ae_dev_resetting(handle);

        if (reset_cnt != hr_dev->reset_cnt)
                return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
                                                  reset_stage);
        else if (hw_resetting)
                return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
                                                    reset_stage);
        else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
                return hns_roce_v2_cmd_sw_resetting(hr_dev);

        return 0;
}

static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
{
        int ntu = ring->next_to_use;
        int ntc = ring->next_to_clean;
        int used = (ntu - ntc + ring->desc_num) % ring->desc_num;

        return ring->desc_num - used - 1;
}

static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
                                   struct hns_roce_v2_cmq_ring *ring)
{
        int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);

        ring->desc = kzalloc(size, GFP_KERNEL);
        if (!ring->desc)
                return -ENOMEM;

        ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
                                             DMA_BIDIRECTIONAL);
        if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
                ring->desc_dma_addr = 0;
                kfree(ring->desc);
                ring->desc = NULL;
                return -ENOMEM;
        }

        return 0;
}

static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
                                   struct hns_roce_v2_cmq_ring *ring)
{
        dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
                         ring->desc_num * sizeof(struct hns_roce_cmq_desc),
                         DMA_BIDIRECTIONAL);

        ring->desc_dma_addr = 0;
        kfree(ring->desc);
}

static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
                                            &priv->cmq.csq : &priv->cmq.crq;

        ring->flag = ring_type;
        ring->next_to_clean = 0;
        ring->next_to_use = 0;

        return hns_roce_alloc_cmq_desc(hr_dev, ring);
}

static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
                                            &priv->cmq.csq : &priv->cmq.crq;
        dma_addr_t dma = ring->desc_dma_addr;

        if (ring_type == TYPE_CSQ) {
                roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
                roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
                           upper_32_bits(dma));
                roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
                           ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
                roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
                roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
        } else {

                roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
                roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
                           upper_32_bits(dma));
                roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
                           ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
                roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
                roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
        }
}

static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        int ret;

        /* Setup the queue entries for command queue */
        priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
        priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;

        /* Setup the lock for command queue */
        spin_lock_init(&priv->cmq.csq.lock);
        spin_lock_init(&priv->cmq.crq.lock);

        /* Setup Tx write back timeout */
        priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;

        /* Init CSQ */
        ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
        if (ret) {
                dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
                return ret;
        }

        /* Init CRQ */
        ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
        if (ret) {
                dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
                goto err_crq;
        }

        /* Init CSQ REG */
        hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);

        /* Init CRQ REG */
        hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);

        return 0;

err_crq:
        hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);

        return ret;
}

static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;

        hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
        hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
}

static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
                                          enum hns_roce_opcode_type opcode,
                                          bool is_read)
{
        memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
        desc->opcode = cpu_to_le16(opcode);
        desc->flag =
                cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
        if (is_read)
                desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
        else
                desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
}

static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
{
        u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
        struct hns_roce_v2_priv *priv = hr_dev->priv;

        return head == priv->cmq.csq.next_to_use;
}

static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
        struct hns_roce_cmq_desc *desc;
        u16 ntc = csq->next_to_clean;
        u32 head;
        int clean = 0;

        desc = &csq->desc[ntc];
        head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
        while (head != ntc) {
                memset(desc, 0, sizeof(*desc));
                ntc++;
                if (ntc == csq->desc_num)
                        ntc = 0;
                desc = &csq->desc[ntc];
                clean++;
        }
        csq->next_to_clean = ntc;

        return clean;
}

static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
                               struct hns_roce_cmq_desc *desc, int num)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
        struct hns_roce_cmq_desc *desc_to_use;
        bool complete = false;
        u32 timeout = 0;
        int handle = 0;
        u16 desc_ret;
        int ret = 0;
        int ntc;

        spin_lock_bh(&csq->lock);

        if (num > hns_roce_cmq_space(csq)) {
                spin_unlock_bh(&csq->lock);
                return -EBUSY;
        }

        /*
         * Record the location of desc in the cmq for this time
         * which will be use for hardware to write back
         */
        ntc = csq->next_to_use;

        while (handle < num) {
                desc_to_use = &csq->desc[csq->next_to_use];
                *desc_to_use = desc[handle];
                dev_dbg(hr_dev->dev, "set cmq desc:\n");
                csq->next_to_use++;
                if (csq->next_to_use == csq->desc_num)
                        csq->next_to_use = 0;
                handle++;
        }

        /* Write to hardware */
        roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);

        /*
         * If the command is sync, wait for the firmware to write back,
         * if multi descriptors to be sent, use the first one to check
         */
        if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
                do {
                        if (hns_roce_cmq_csq_done(hr_dev))
                                break;
                        udelay(1);
                        timeout++;
                } while (timeout < priv->cmq.tx_timeout);
        }

        if (hns_roce_cmq_csq_done(hr_dev)) {
                complete = true;
                handle = 0;
                while (handle < num) {
                        /* get the result of hardware write back */
                        desc_to_use = &csq->desc[ntc];
                        desc[handle] = *desc_to_use;
                        dev_dbg(hr_dev->dev, "Get cmq desc:\n");
                        desc_ret = le16_to_cpu(desc[handle].retval);
                        if (desc_ret == CMD_EXEC_SUCCESS)
                                ret = 0;
                        else
                                ret = -EIO;
                        priv->cmq.last_status = desc_ret;
                        ntc++;
                        handle++;
                        if (ntc == csq->desc_num)
                                ntc = 0;
                }
        }

        if (!complete)
                ret = -EAGAIN;

        /* clean the command send queue */
        handle = hns_roce_cmq_csq_clean(hr_dev);
        if (handle != num)
                dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
                         handle, num);

        spin_unlock_bh(&csq->lock);

        return ret;
}

static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
                             struct hns_roce_cmq_desc *desc, int num)
{
        int retval;
        int ret;

        ret = hns_roce_v2_rst_process_cmd(hr_dev);
        if (ret == CMD_RST_PRC_SUCCESS)
                return 0;
        if (ret == CMD_RST_PRC_EBUSY)
                return -EBUSY;

        ret = __hns_roce_cmq_send(hr_dev, desc, num);
        if (ret) {
                retval = hns_roce_v2_rst_process_cmd(hr_dev);
                if (retval == CMD_RST_PRC_SUCCESS)
                        return 0;
                else if (retval == CMD_RST_PRC_EBUSY)
                        return -EBUSY;
        }

        return ret;
}

static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_query_version *resp;
        struct hns_roce_cmq_desc desc;
        int ret;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
        ret = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (ret)
                return ret;

        resp = (struct hns_roce_query_version *)desc.data;
        hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
        hr_dev->vendor_id = hr_dev->pci_dev->vendor;

        return 0;
}

static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hnae3_handle *handle = priv->handle;
        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
        unsigned long reset_cnt;
        bool sw_resetting;
        bool hw_resetting;

        reset_cnt = ops->ae_dev_reset_cnt(handle);
        hw_resetting = ops->get_hw_reset_stat(handle);
        sw_resetting = ops->ae_dev_resetting(handle);

        if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting)
                return true;

        return false;
}

static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
                                      int flag)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hnae3_handle *handle = priv->handle;
        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
        unsigned long instance_stage;
        unsigned long reset_cnt;
        unsigned long end;
        bool sw_resetting;
        bool hw_resetting;

        instance_stage = handle->rinfo.instance_state;
        reset_cnt = ops->ae_dev_reset_cnt(handle);
        hw_resetting = ops->get_hw_reset_stat(handle);
        sw_resetting = ops->ae_dev_resetting(handle);

        if (reset_cnt != hr_dev->reset_cnt) {
                hr_dev->dis_db = true;
                hr_dev->is_reset = true;
                dev_info(hr_dev->dev, "Func clear success after reset.\n");
        } else if (hw_resetting) {
                hr_dev->dis_db = true;

                dev_warn(hr_dev->dev,
                         "Func clear is pending, device in resetting state.\n");
                end = HNS_ROCE_V2_HW_RST_TIMEOUT;
                while (end) {
                        if (!ops->get_hw_reset_stat(handle)) {
                                hr_dev->is_reset = true;
                                dev_info(hr_dev->dev,
                                         "Func clear success after reset.\n");
                                return;
                        }
                        msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
                        end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
                }

                dev_warn(hr_dev->dev, "Func clear failed.\n");
        } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
                hr_dev->dis_db = true;

                dev_warn(hr_dev->dev,
                         "Func clear is pending, device in resetting state.\n");
                end = HNS_ROCE_V2_HW_RST_TIMEOUT;
                while (end) {
                        if (ops->ae_dev_reset_cnt(handle) !=
                            hr_dev->reset_cnt) {
                                hr_dev->is_reset = true;
                                dev_info(hr_dev->dev,
                                         "Func clear success after sw reset\n");
                                return;
                        }
                        msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
                        end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
                }

                dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
        } else {
                if (retval && !flag)
                        dev_warn(hr_dev->dev,
                                 "Func clear read failed, ret = %d.\n", retval);

                dev_warn(hr_dev->dev, "Func clear failed.\n");
        }
}
static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
{
        bool fclr_write_fail_flag = false;
        struct hns_roce_func_clear *resp;
        struct hns_roce_cmq_desc desc;
        unsigned long end;
        int ret = 0;

        if (hns_roce_func_clr_chk_rst(hr_dev))
                goto out;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
        resp = (struct hns_roce_func_clear *)desc.data;

        ret = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (ret) {
                fclr_write_fail_flag = true;
                dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
                         ret);
                goto out;
        }

        msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
        end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
        while (end) {
                if (hns_roce_func_clr_chk_rst(hr_dev))
                        goto out;
                msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
                end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;

                hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
                                              true);

                ret = hns_roce_cmq_send(hr_dev, &desc, 1);
                if (ret)
                        continue;

                if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
                        hr_dev->is_reset = true;
                        return;
                }
        }

out:
        hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
}

static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_query_fw_info *resp;
        struct hns_roce_cmq_desc desc;
        int ret;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
        ret = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (ret)
                return ret;

        resp = (struct hns_roce_query_fw_info *)desc.data;
        hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));

        return 0;
}

static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_cfg_global_param *req;
        struct hns_roce_cmq_desc desc;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
                                      false);

        req = (struct hns_roce_cfg_global_param *)desc.data;
        memset(req, 0, sizeof(*req));
        roce_set_field(req->time_cfg_udp_port,
                       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
                       CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
        roce_set_field(req->time_cfg_udp_port,
                       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
                       CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_cmq_desc desc[2];
        struct hns_roce_pf_res_a *req_a;
        struct hns_roce_pf_res_b *req_b;
        int ret;
        int i;

        for (i = 0; i < 2; i++) {
                hns_roce_cmq_setup_basic_desc(&desc[i],
                                              HNS_ROCE_OPC_QUERY_PF_RES, true);

                if (i == 0)
                        desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
                else
                        desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
        }

        ret = hns_roce_cmq_send(hr_dev, desc, 2);
        if (ret)
                return ret;

        req_a = (struct hns_roce_pf_res_a *)desc[0].data;
        req_b = (struct hns_roce_pf_res_b *)desc[1].data;

        hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
                                                 PF_RES_DATA_1_PF_QPC_BT_NUM_M,
                                                 PF_RES_DATA_1_PF_QPC_BT_NUM_S);
        hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
                                                PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
                                                PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
        hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
                                                 PF_RES_DATA_3_PF_CQC_BT_NUM_M,
                                                 PF_RES_DATA_3_PF_CQC_BT_NUM_S);
        hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
                                                 PF_RES_DATA_4_PF_MPT_BT_NUM_M,
                                                 PF_RES_DATA_4_PF_MPT_BT_NUM_S);

        hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
                                             PF_RES_DATA_3_PF_SL_NUM_M,
                                             PF_RES_DATA_3_PF_SL_NUM_S);
        hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
                                             PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
                                             PF_RES_DATA_4_PF_SCCC_BT_NUM_S);

        return 0;
}

static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_pf_timer_res_a *req_a;
        struct hns_roce_cmq_desc desc;
        int ret;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
                                      true);

        ret = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (ret)
                return ret;

        req_a = (struct hns_roce_pf_timer_res_a *)desc.data;

        hr_dev->caps.qpc_timer_bt_num =
                roce_get_field(req_a->qpc_timer_bt_idx_num,
                               PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
                               PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
        hr_dev->caps.cqc_timer_bt_num =
                roce_get_field(req_a->cqc_timer_bt_idx_num,
                               PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
                               PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);

        return 0;
}

static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
{
        struct hns_roce_cmq_desc desc;
        struct hns_roce_vf_switch *swt;
        int ret;

        swt = (struct hns_roce_vf_switch *)desc.data;
        hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
        swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
        roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
                       VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
        ret = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (ret)
                return ret;

        desc.flag =
                cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
        desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
        roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
        roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
        roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_cmq_desc desc[2];
        struct hns_roce_vf_res_a *req_a;
        struct hns_roce_vf_res_b *req_b;
        int i;

        req_a = (struct hns_roce_vf_res_a *)desc[0].data;
        req_b = (struct hns_roce_vf_res_b *)desc[1].data;
        for (i = 0; i < 2; i++) {
                hns_roce_cmq_setup_basic_desc(&desc[i],
                                              HNS_ROCE_OPC_ALLOC_VF_RES, false);

                if (i == 0)
                        desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
                else
                        desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
        }

        roce_set_field(req_a->vf_qpc_bt_idx_num,
                       VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
                       VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
        roce_set_field(req_a->vf_qpc_bt_idx_num,
                       VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
                       VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM);

        roce_set_field(req_a->vf_srqc_bt_idx_num,
                       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
                       VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
        roce_set_field(req_a->vf_srqc_bt_idx_num,
                       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
                       VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
                       HNS_ROCE_VF_SRQC_BT_NUM);

        roce_set_field(req_a->vf_cqc_bt_idx_num,
                       VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
                       VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
        roce_set_field(req_a->vf_cqc_bt_idx_num,
                       VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
                       VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM);

        roce_set_field(req_a->vf_mpt_bt_idx_num,
                       VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
                       VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
        roce_set_field(req_a->vf_mpt_bt_idx_num,
                       VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
                       VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM);

        roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M,
                       VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
        roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M,
                       VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM);

        roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M,
                       VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
        roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M,
                       VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM);

        roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M,
                       VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
        roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M,
                       VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM);

        roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M,
                       VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
        roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M,
                       VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM);

        roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
                       VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
        roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
                       VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
                       HNS_ROCE_VF_SCCC_BT_NUM);

        return hns_roce_cmq_send(hr_dev, desc, 2);
}

static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
{
        u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
        u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
        u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
        u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
        u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
        struct hns_roce_cfg_bt_attr *req;
        struct hns_roce_cmq_desc desc;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
        req = (struct hns_roce_cfg_bt_attr *)desc.data;
        memset(req, 0, sizeof(*req));

        roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
                       CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
                       hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
                       CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
                       hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
                       CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
                       qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);

        roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
                       CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
                       hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
                       CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
                       hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
                       CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
                       srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);

        roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
                       CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
                       hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
                       CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
                       hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
                       CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
                       cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);

        roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
                       CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
                       hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
                       CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
                       hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
                       CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
                       mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);

        roce_set_field(req->vf_sccc_cfg,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
                       hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_sccc_cfg,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
                       hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
        roce_set_field(req->vf_sccc_cfg,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
                       CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
                       sccc_hop_num ==
                              HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static void set_default_caps(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_caps *caps = &hr_dev->caps;

        caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
        caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
        caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
        caps->num_srqs          = HNS_ROCE_V2_MAX_SRQ_NUM;
        caps->min_cqes          = HNS_ROCE_MIN_CQE_NUM;
        caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
        caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
        caps->max_extend_sg     = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
        caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
        caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
        caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
        caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
        caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
        caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
        caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
        caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
        caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
        caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
        caps->num_srqwqe_segs   = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
        caps->num_idx_segs      = HNS_ROCE_V2_MAX_IDX_SEGS;
        caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
        caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
        caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
        caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
        caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
        caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
        caps->qpc_entry_sz      = HNS_ROCE_V2_QPC_ENTRY_SZ;
        caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
        caps->trrl_entry_sz     = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
        caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
        caps->srqc_entry_sz     = HNS_ROCE_V2_SRQC_ENTRY_SZ;
        caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
        caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
        caps->idx_entry_sz      = HNS_ROCE_V2_IDX_ENTRY_SZ;
        caps->cq_entry_sz       = HNS_ROCE_V2_CQE_ENTRY_SIZE;
        caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
        caps->reserved_lkey     = 0;
        caps->reserved_pds      = 0;
        caps->reserved_mrws     = 1;
        caps->reserved_uars     = 0;
        caps->reserved_cqs      = 0;
        caps->reserved_srqs     = 0;
        caps->reserved_qps      = HNS_ROCE_V2_RSV_QPS;

        caps->qpc_ba_pg_sz      = 0;
        caps->qpc_buf_pg_sz     = 0;
        caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
        caps->srqc_ba_pg_sz     = 0;
        caps->srqc_buf_pg_sz    = 0;
        caps->srqc_hop_num      = HNS_ROCE_CONTEXT_HOP_NUM;
        caps->cqc_ba_pg_sz      = 0;
        caps->cqc_buf_pg_sz     = 0;
        caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
        caps->mpt_ba_pg_sz      = 0;
        caps->mpt_buf_pg_sz     = 0;
        caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
        caps->mtt_ba_pg_sz      = 0;
        caps->mtt_buf_pg_sz     = 0;
        caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
        caps->wqe_sq_hop_num    = HNS_ROCE_SQWQE_HOP_NUM;
        caps->wqe_sge_hop_num   = HNS_ROCE_EXT_SGE_HOP_NUM;
        caps->wqe_rq_hop_num    = HNS_ROCE_RQWQE_HOP_NUM;
        caps->cqe_ba_pg_sz      = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
        caps->cqe_buf_pg_sz     = 0;
        caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
        caps->srqwqe_ba_pg_sz   = 0;
        caps->srqwqe_buf_pg_sz  = 0;
        caps->srqwqe_hop_num    = HNS_ROCE_SRQWQE_HOP_NUM;
        caps->idx_ba_pg_sz      = 0;
        caps->idx_buf_pg_sz     = 0;
        caps->idx_hop_num       = HNS_ROCE_IDX_HOP_NUM;
        caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;

        caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
                                  HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
                                  HNS_ROCE_CAP_FLAG_RQ_INLINE |
                                  HNS_ROCE_CAP_FLAG_RECORD_DB |
                                  HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;

        caps->pkey_table_len[0] = 1;
        caps->gid_table_len[0]  = HNS_ROCE_V2_GID_INDEX_NUM;
        caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
        caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
        caps->local_ca_ack_delay = 0;
        caps->max_mtu = IB_MTU_4096;

        caps->max_srq_wrs       = HNS_ROCE_V2_MAX_SRQ_WR;
        caps->max_srq_sges      = HNS_ROCE_V2_MAX_SRQ_SGE;

        caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
                       HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
                       HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;

        caps->num_qpc_timer       = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
        caps->qpc_timer_entry_sz  = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
        caps->qpc_timer_ba_pg_sz  = 0;
        caps->qpc_timer_buf_pg_sz = 0;
        caps->qpc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
        caps->num_cqc_timer       = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
        caps->cqc_timer_entry_sz  = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
        caps->cqc_timer_ba_pg_sz  = 0;
        caps->cqc_timer_buf_pg_sz = 0;
        caps->cqc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;

        caps->sccc_entry_sz       = HNS_ROCE_V2_SCCC_ENTRY_SZ;
        caps->sccc_ba_pg_sz       = 0;
        caps->sccc_buf_pg_sz      = 0;
        caps->sccc_hop_num        = HNS_ROCE_SCCC_HOP_NUM;
}

static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
                       int *buf_page_size, int *bt_page_size, u32 hem_type)
{
        u64 obj_per_chunk;
        int bt_chunk_size = 1 << PAGE_SHIFT;
        int buf_chunk_size = 1 << PAGE_SHIFT;
        int obj_per_chunk_default = buf_chunk_size / obj_size;

        *buf_page_size = 0;
        *bt_page_size = 0;

        switch (hop_num) {
        case 3:
                obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
                                (bt_chunk_size / BA_BYTE_LEN) *
                                (bt_chunk_size / BA_BYTE_LEN) *
                                 obj_per_chunk_default;
                break;
        case 2:
                obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
                                (bt_chunk_size / BA_BYTE_LEN) *
                                 obj_per_chunk_default;
                break;
        case 1:
                obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
                                obj_per_chunk_default;
                break;
        case HNS_ROCE_HOP_NUM_0:
                obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
                break;
        default:
                pr_err("Table %d not support hop_num = %d!\n", hem_type,
                        hop_num);
                return;
        }

        if (hem_type >= HEM_TYPE_MTT)
                *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
        else
                *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
}

static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
        struct hns_roce_caps *caps = &hr_dev->caps;
        struct hns_roce_query_pf_caps_a *resp_a;
        struct hns_roce_query_pf_caps_b *resp_b;
        struct hns_roce_query_pf_caps_c *resp_c;
        struct hns_roce_query_pf_caps_d *resp_d;
        struct hns_roce_query_pf_caps_e *resp_e;
        int ctx_hop_num;
        int pbl_hop_num;
        int ret;
        int i;

        for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
                hns_roce_cmq_setup_basic_desc(&desc[i],
                                              HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
                                              true);
                if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
                        desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
                else
                        desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
        }

        ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
        if (ret)
                return ret;

        resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
        resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
        resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
        resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
        resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;

        caps->local_ca_ack_delay     = resp_a->local_ca_ack_delay;
        caps->max_sq_sg              = le16_to_cpu(resp_a->max_sq_sg);
        caps->max_sq_inline          = le16_to_cpu(resp_a->max_sq_inline);
        caps->max_rq_sg              = le16_to_cpu(resp_a->max_rq_sg);
        caps->max_extend_sg          = le32_to_cpu(resp_a->max_extend_sg);
        caps->num_qpc_timer          = le16_to_cpu(resp_a->num_qpc_timer);
        caps->num_cqc_timer          = le16_to_cpu(resp_a->num_cqc_timer);
        caps->max_srq_sges           = le16_to_cpu(resp_a->max_srq_sges);
        caps->num_aeq_vectors        = resp_a->num_aeq_vectors;
        caps->num_other_vectors      = resp_a->num_other_vectors;
        caps->max_sq_desc_sz         = resp_a->max_sq_desc_sz;
        caps->max_rq_desc_sz         = resp_a->max_rq_desc_sz;
        caps->max_srq_desc_sz        = resp_a->max_srq_desc_sz;
        caps->cq_entry_sz            = resp_a->cq_entry_sz;

        caps->mtpt_entry_sz          = resp_b->mtpt_entry_sz;
        caps->irrl_entry_sz          = resp_b->irrl_entry_sz;
        caps->trrl_entry_sz          = resp_b->trrl_entry_sz;
        caps->cqc_entry_sz           = resp_b->cqc_entry_sz;
        caps->srqc_entry_sz          = resp_b->srqc_entry_sz;
        caps->idx_entry_sz           = resp_b->idx_entry_sz;
        caps->sccc_entry_sz          = resp_b->scc_ctx_entry_sz;
        caps->max_mtu                = resp_b->max_mtu;
        caps->qpc_entry_sz           = le16_to_cpu(resp_b->qpc_entry_sz);
        caps->min_cqes               = resp_b->min_cqes;
        caps->min_wqes               = resp_b->min_wqes;
        caps->page_size_cap          = le32_to_cpu(resp_b->page_size_cap);
        caps->pkey_table_len[0]      = resp_b->pkey_table_len;
        caps->phy_num_uars           = resp_b->phy_num_uars;
        ctx_hop_num                  = resp_b->ctx_hop_num;
        pbl_hop_num                  = resp_b->pbl_hop_num;

        caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
                                            V2_QUERY_PF_CAPS_C_NUM_PDS_M,
                                            V2_QUERY_PF_CAPS_C_NUM_PDS_S);
        caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
                                     V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
                                     V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
        caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
                       HNS_ROCE_CAP_FLAGS_EX_SHIFT;

        caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
                                            V2_QUERY_PF_CAPS_C_NUM_CQS_M,
                                            V2_QUERY_PF_CAPS_C_NUM_CQS_S);
        caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
                                                V2_QUERY_PF_CAPS_C_MAX_GID_M,
                                                V2_QUERY_PF_CAPS_C_MAX_GID_S);
        caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
                                             V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
                                             V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
        caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
                                              V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
                                              V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
        caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
                                            V2_QUERY_PF_CAPS_C_NUM_QPS_M,
                                            V2_QUERY_PF_CAPS_C_NUM_QPS_S);
        caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
                                                V2_QUERY_PF_CAPS_C_MAX_ORD_M,
                                                V2_QUERY_PF_CAPS_C_MAX_ORD_S);
        caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
        caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
        caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
                                             V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
                                             V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
        caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
        caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
                                               V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
                                               V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
        caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
                                                V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
                                                V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
        caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
                                               V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
                                               V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
        caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
                                            V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
                                            V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
        caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
                                            V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
                                            V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
        caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
                                            V2_QUERY_PF_CAPS_D_RSV_PDS_M,
                                            V2_QUERY_PF_CAPS_D_RSV_PDS_S);
        caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
                                             V2_QUERY_PF_CAPS_D_NUM_UARS_M,
                                             V2_QUERY_PF_CAPS_D_NUM_UARS_S);
        caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
                                            V2_QUERY_PF_CAPS_D_RSV_QPS_M,
                                            V2_QUERY_PF_CAPS_D_RSV_QPS_S);
        caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
                                             V2_QUERY_PF_CAPS_D_RSV_UARS_M,
                                             V2_QUERY_PF_CAPS_D_RSV_UARS_S);
        caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
                                             V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
                                             V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
        caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
                                         V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
                                         V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
        caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
                                            V2_QUERY_PF_CAPS_E_RSV_CQS_M,
                                            V2_QUERY_PF_CAPS_E_RSV_CQS_S);
        caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
                                             V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
                                             V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
        caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
                                             V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
                                             V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
        caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
        caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
        caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
        caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);

        caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
        caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
        caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
        caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
        caps->mtt_ba_pg_sz = 0;
        caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
        caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
        caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;

        caps->qpc_hop_num = ctx_hop_num;
        caps->srqc_hop_num = ctx_hop_num;
        caps->cqc_hop_num = ctx_hop_num;
        caps->mpt_hop_num = ctx_hop_num;
        caps->mtt_hop_num = pbl_hop_num;
        caps->cqe_hop_num = pbl_hop_num;
        caps->srqwqe_hop_num = pbl_hop_num;
        caps->idx_hop_num = pbl_hop_num;
        caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
                                          V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
                                          V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
        caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
                                          V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
                                          V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
        caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
                                          V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
                                          V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);

        calc_pg_sz(caps->num_qps, caps->qpc_entry_sz, caps->qpc_hop_num,
                   caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
                   HEM_TYPE_QPC);
        calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
                   caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
                   HEM_TYPE_MTPT);
        calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
                   caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
                   HEM_TYPE_CQC);
        calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num,
                   caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
                   &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);

        caps->sccc_hop_num = ctx_hop_num;
        caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
        caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;

        calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,
                   caps->sccc_hop_num, caps->sccc_bt_num,
                   &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
                   HEM_TYPE_SCCC);
        calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
                   caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
                   &caps->cqc_timer_buf_pg_sz,
                   &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);

        calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
                   1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
        calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
                   caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
                   &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
        calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num,
                   1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX);

        return 0;
}

static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_caps *caps = &hr_dev->caps;
        int ret;

        ret = hns_roce_cmq_query_hw_info(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
                        ret);
                return ret;
        }

        ret = hns_roce_query_fw_ver(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
                        ret);
                return ret;
        }

        ret = hns_roce_config_global_param(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
                        ret);
                return ret;
        }

        /* Get pf resource owned by every pf */
        ret = hns_roce_query_pf_resource(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
                        ret);
                return ret;
        }

        ret = hns_roce_query_pf_timer_resource(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev,
                        "failed to query pf timer resource, ret = %d.\n", ret);
                return ret;
        }

        ret = hns_roce_set_vf_switch_param(hr_dev, 0);
        if (ret) {
                dev_err(hr_dev->dev,
                        "failed to set function switch param, ret = %d.\n",
                        ret);
                return ret;
        }

        hr_dev->vendor_part_id = hr_dev->pci_dev->device;
        hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);

        caps->pbl_ba_pg_sz      = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
        caps->pbl_buf_pg_sz     = 0;
        caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
        caps->eqe_ba_pg_sz      = 0;
        caps->eqe_buf_pg_sz     = 0;
        caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
        caps->tsq_buf_pg_sz     = 0;

        ret = hns_roce_query_pf_caps(hr_dev);
        if (ret)
                set_default_caps(hr_dev);

        ret = hns_roce_alloc_vf_resource(hr_dev);
        if (ret) {
                dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
                        ret);
                return ret;
        }

        ret = hns_roce_v2_set_bt(hr_dev);
        if (ret)
                dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
                        ret);

        return ret;
}

static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
                                      enum hns_roce_link_table_type type)
{
        struct hns_roce_cmq_desc desc[2];
        struct hns_roce_cfg_llm_a *req_a =
                                (struct hns_roce_cfg_llm_a *)desc[0].data;
        struct hns_roce_cfg_llm_b *req_b =
                                (struct hns_roce_cfg_llm_b *)desc[1].data;
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_link_table *link_tbl;
        struct hns_roce_link_table_entry *entry;
        enum hns_roce_opcode_type opcode;
        u32 page_num;
        int i;

        switch (type) {
        case TSQ_LINK_TABLE:
                link_tbl = &priv->tsq;
                opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
                break;
        case TPQ_LINK_TABLE:
                link_tbl = &priv->tpq;
                opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
                break;
        default:
                return -EINVAL;
        }

        page_num = link_tbl->npages;
        entry = link_tbl->table.buf;

        for (i = 0; i < 2; i++) {
                hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);

                if (i == 0)
                        desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
                else
                        desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
        }

        req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff);
        req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32);
        roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M,
                       CFG_LLM_QUE_DEPTH_S, link_tbl->npages);
        roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M,
                       CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz);
        roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
                       CFG_LLM_INIT_EN_S, 1);
        req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
        req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
        roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S,
                       0);

        req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0);
        roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
                       CFG_LLM_TAIL_BA_H_S,
                       entry[page_num - 1].blk_ba1_nxt_ptr &
                       HNS_ROCE_LINK_TABLE_BA1_M);
        roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S,
                       (entry[page_num - 2].blk_ba1_nxt_ptr &
                        HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
                        HNS_ROCE_LINK_TABLE_NXT_PTR_S);

        return hns_roce_cmq_send(hr_dev, desc, 2);
}

static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
                                    enum hns_roce_link_table_type type)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        struct hns_roce_link_table *link_tbl;
        struct hns_roce_link_table_entry *entry;
        struct device *dev = hr_dev->dev;
        u32 buf_chk_sz;
        dma_addr_t t;
        int func_num = 1;
        int pg_num_a;
        int pg_num_b;
        int pg_num;
        int size;
        int i;

        switch (type) {
        case TSQ_LINK_TABLE:
                link_tbl = &priv->tsq;
                buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
                pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
                pg_num_b = hr_dev->caps.sl_num * 4 + 2;
                break;
        case TPQ_LINK_TABLE:
                link_tbl = &priv->tpq;
                buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
                pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
                pg_num_b = 2 * 4 * func_num + 2;
                break;
        default:
                return -EINVAL;
        }

        pg_num = max(pg_num_a, pg_num_b);
        size = pg_num * sizeof(struct hns_roce_link_table_entry);

        link_tbl->table.buf = dma_alloc_coherent(dev, size,
                                                 &link_tbl->table.map,
                                                 GFP_KERNEL);
        if (!link_tbl->table.buf)
                goto out;

        link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
                                    GFP_KERNEL);
        if (!link_tbl->pg_list)
                goto err_kcalloc_failed;

        entry = link_tbl->table.buf;
        for (i = 0; i < pg_num; ++i) {
                link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
                                                              &t, GFP_KERNEL);
                if (!link_tbl->pg_list[i].buf)
                        goto err_alloc_buf_failed;

                link_tbl->pg_list[i].map = t;

                entry[i].blk_ba0 = (u32)(t >> 12);
                entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);

                if (i < (pg_num - 1))
                        entry[i].blk_ba1_nxt_ptr |=
                                (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;

        }
        link_tbl->npages = pg_num;
        link_tbl->pg_sz = buf_chk_sz;

        return hns_roce_config_link_table(hr_dev, type);

err_alloc_buf_failed:
        for (i -= 1; i >= 0; i--)
                dma_free_coherent(dev, buf_chk_sz,
                                  link_tbl->pg_list[i].buf,
                                  link_tbl->pg_list[i].map);
        kfree(link_tbl->pg_list);

err_kcalloc_failed:
        dma_free_coherent(dev, size, link_tbl->table.buf,
                          link_tbl->table.map);

out:
        return -ENOMEM;
}

static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
                                     struct hns_roce_link_table *link_tbl)
{
        struct device *dev = hr_dev->dev;
        int size;
        int i;

        size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);

        for (i = 0; i < link_tbl->npages; ++i)
                if (link_tbl->pg_list[i].buf)
                        dma_free_coherent(dev, link_tbl->pg_sz,
                                          link_tbl->pg_list[i].buf,
                                          link_tbl->pg_list[i].map);
        kfree(link_tbl->pg_list);

        dma_free_coherent(dev, size, link_tbl->table.buf,
                          link_tbl->table.map);
}

static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;
        int qpc_count, cqc_count;
        int ret, i;

        /* TSQ includes SQ doorbell and ack doorbell */
        ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
        if (ret) {
                dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
                return ret;
        }

        ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
        if (ret) {
                dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
                goto err_tpq_init_failed;
        }

        /* Alloc memory for QPC Timer buffer space chunk */
        for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
             qpc_count++) {
                ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
                                         qpc_count);
                if (ret) {
                        dev_err(hr_dev->dev, "QPC Timer get failed\n");
                        goto err_qpc_timer_failed;
                }
        }

        /* Alloc memory for CQC Timer buffer space chunk */
        for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
             cqc_count++) {
                ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
                                         cqc_count);
                if (ret) {
                        dev_err(hr_dev->dev, "CQC Timer get failed\n");
                        goto err_cqc_timer_failed;
                }
        }

        return 0;

err_cqc_timer_failed:
        for (i = 0; i < cqc_count; i++)
                hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);

err_qpc_timer_failed:
        for (i = 0; i < qpc_count; i++)
                hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);

        hns_roce_free_link_table(hr_dev, &priv->tpq);

err_tpq_init_failed:
        hns_roce_free_link_table(hr_dev, &priv->tsq);

        return ret;
}

static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_v2_priv *priv = hr_dev->priv;

        hns_roce_function_clear(hr_dev);

        hns_roce_free_link_table(hr_dev, &priv->tpq);
        hns_roce_free_link_table(hr_dev, &priv->tsq);
}

static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
{
        struct hns_roce_cmq_desc desc;
        struct hns_roce_mbox_status *mb_st =
                                       (struct hns_roce_mbox_status *)desc.data;
        enum hns_roce_cmd_return_status status;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);

        status = hns_roce_cmq_send(hr_dev, &desc, 1);
        if (status)
                return status;

        return le32_to_cpu(mb_st->mb_status_hw_run);
}

static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
{
        u32 status = hns_roce_query_mbox_status(hr_dev);

        return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
}

static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
{
        u32 status = hns_roce_query_mbox_status(hr_dev);

        return status & HNS_ROCE_HW_MB_STATUS_MASK;
}

static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
                              u64 out_param, u32 in_modifier, u8 op_modifier,
                              u16 op, u16 token, int event)
{
        struct hns_roce_cmq_desc desc;
        struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);

        mb->in_param_l = cpu_to_le32(in_param);
        mb->in_param_h = cpu_to_le32(in_param >> 32);
        mb->out_param_l = cpu_to_le32(out_param);
        mb->out_param_h = cpu_to_le32(out_param >> 32);
        mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
        mb->token_event_en = cpu_to_le32(event << 16 | token);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
                                 u64 out_param, u32 in_modifier, u8 op_modifier,
                                 u16 op, u16 token, int event)
{
        struct device *dev = hr_dev->dev;
        unsigned long end;
        int ret;

        end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
        while (hns_roce_v2_cmd_pending(hr_dev)) {
                if (time_after(jiffies, end)) {
                        dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
                                (int)end);
                        return -EAGAIN;
                }
                cond_resched();
        }

        ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
                                 op_modifier, op, token, event);
        if (ret)
                dev_err(dev, "Post mailbox fail(%d)\n", ret);

        return ret;
}

static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
                                unsigned long timeout)
{
        struct device *dev = hr_dev->dev;
        unsigned long end;
        u32 status;

        end = msecs_to_jiffies(timeout) + jiffies;
        while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
                cond_resched();

        if (hns_roce_v2_cmd_pending(hr_dev)) {
                dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
                return -ETIMEDOUT;
        }

        status = hns_roce_v2_cmd_complete(hr_dev);
        if (status != 0x1) {
                if (status == CMD_RST_PRC_EBUSY)
                        return status;

                dev_err(dev, "mailbox status 0x%x!\n", status);
                return -EBUSY;
        }

        return 0;
}

static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
                                      int gid_index, const union ib_gid *gid,
                                      enum hns_roce_sgid_type sgid_type)
{
        struct hns_roce_cmq_desc desc;
        struct hns_roce_cfg_sgid_tb *sgid_tb =
                                    (struct hns_roce_cfg_sgid_tb *)desc.data;
        u32 *p;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);

        roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
                       CFG_SGID_TB_TABLE_IDX_S, gid_index);
        roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
                       CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);

        p = (u32 *)&gid->raw[0];
        sgid_tb->vf_sgid_l = cpu_to_le32(*p);

        p = (u32 *)&gid->raw[4];
        sgid_tb->vf_sgid_ml = cpu_to_le32(*p);

        p = (u32 *)&gid->raw[8];
        sgid_tb->vf_sgid_mh = cpu_to_le32(*p);

        p = (u32 *)&gid->raw[0xc];
        sgid_tb->vf_sgid_h = cpu_to_le32(*p);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
                               int gid_index, const union ib_gid *gid,
                               const struct ib_gid_attr *attr)
{
        enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
        int ret;

        if (!gid || !attr)
                return -EINVAL;

        if (attr->gid_type == IB_GID_TYPE_ROCE)
                sgid_type = GID_TYPE_FLAG_ROCE_V1;

        if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
                if (ipv6_addr_v4mapped((void *)gid))
                        sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
                else
                        sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
        }

        ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
        if (ret)
                ibdev_err(&hr_dev->ib_dev,
                          "failed to configure sgid table, ret = %d!\n",
                          ret);

        return ret;
}

static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
                               u8 *addr)
{
        struct hns_roce_cmq_desc desc;
        struct hns_roce_cfg_smac_tb *smac_tb =
                                    (struct hns_roce_cfg_smac_tb *)desc.data;
        u16 reg_smac_h;
        u32 reg_smac_l;

        hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);

        reg_smac_l = *(u32 *)(&addr[0]);
        reg_smac_h = *(u16 *)(&addr[4]);

        roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
                       CFG_SMAC_TB_IDX_S, phy_port);
        roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
                       CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
        smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);

        return hns_roce_cmq_send(hr_dev, &desc, 1);
}

static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
                        struct hns_roce_v2_mpt_entry *mpt_entry,
                        struct hns_roce_mr *mr)
{
        u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
        struct ib_device *ibdev = &hr_dev->ib_dev;
        dma_addr_t pbl_ba;
        int i, count;

        count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
                                  ARRAY_SIZE(pages), &pbl_ba);
        if (count < 1) {
                ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
                          count);
                return -ENOBUFS;
        }

        /* Aligned to the hardware address access unit */
        for (i = 0; i < count; i++)
                pages[i] >>= 6;

        mpt_entry->pbl_size = cpu_to_le32(mr->npages);
        mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
        roce_set_field(mpt_entry->byte_48_mode_ba,
                       V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
                       upper_32_bits(pbl_ba >> 3));

        mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
        roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
                       V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));

        mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
        roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
                       V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
        roce_set_field(mpt_entry->byte_64_buf_pa1,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
                       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));

        return 0;
}

static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
                                  void *mb_buf, struct hns_roce_mr *mr,
                                  unsigned long mtpt_idx)
{
        struct hns_roce_v2_mpt_entry *mpt_entry;
        int ret;

        mpt_entry = mb_buf;
        memset(mpt_entry, 0, sizeof(*mpt_entry));

        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
                       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
                       V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
                       HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
        roce_set_field(mpt_entry->byte_4_pd_hop_st,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
                       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
                       V2_MPT_BYTE_4_PD_S, mr->pd);

        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
                     (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
                     mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
                     (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
                     (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
                     (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));

        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
                     mr->type == MR_TYPE_MR ? 0 : 1);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
                     1);

        mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
        mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
        mpt_entry->lkey = cpu_to_le32(mr->key);
        mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
        mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));

        if (mr->type == MR_TYPE_DMA)
                return 0;

        ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);

        return ret;
}

static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
                                        struct hns_roce_mr *mr, int flags,
                                        u32 pdn, int mr_access_flags, u64 iova,
                                        u64 size, void *mb_buf)
{
        struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
        int ret = 0;

        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
                       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);

        if (flags & IB_MR_REREG_PD) {
                roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
                               V2_MPT_BYTE_4_PD_S, pdn);
                mr->pd = pdn;
        }

        if (flags & IB_MR_REREG_ACCESS) {
                roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
                             V2_MPT_BYTE_8_BIND_EN_S,
                             (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
                roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
                             V2_MPT_BYTE_8_ATOMIC_EN_S,
                             mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
                roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
                             mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
                roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
                             mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
                roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
                             mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
        }

        if (flags & IB_MR_REREG_TRANS) {
                mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
                mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
                mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
                mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));

                mr->iova = iova;
                mr->size = size;

                ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
        }

        return ret;
}

static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
                                       void *mb_buf, struct hns_roce_mr *mr)
{
        struct ib_device *ibdev = &hr_dev->ib_dev;
        struct hns_roce_v2_mpt_entry *mpt_entry;
        dma_addr_t pbl_ba = 0;

        mpt_entry = mb_buf;
        memset(mpt_entry, 0, sizeof(*mpt_entry));

        if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
                ibdev_err(ibdev, "failed to find frmr mtr.\n");
                return -ENOBUFS;
        }

        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
                       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
                       V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
        roce_set_field(mpt_entry->byte_4_pd_hop_st,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
                       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
                       V2_MPT_BYTE_4_PD_S, mr->pd);

        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);

        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);

        mpt_entry->pbl_size = cpu_to_le32(mr->npages);

        mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
        roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
                       V2_MPT_BYTE_48_PBL_BA_H_S,
                       upper_32_bits(pbl_ba >> 3));

        roce_set_field(mpt_entry->byte_64_buf_pa1,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
                       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));

        return 0;
}

static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
{
        struct hns_roce_v2_mpt_entry *mpt_entry;

        mpt_entry = mb_buf;
        memset(mpt_entry, 0, sizeof(*mpt_entry));

        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
                       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
                       V2_MPT_BYTE_4_PD_S, mw->pdn);
        roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
                       V2_MPT_BYTE_4_PBL_HOP_NUM_S,
                       mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
                                                               mw->pbl_hop_num);
        roce_set_field(mpt_entry->byte_4_pd_hop_st,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
                       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
                       mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);

        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
        roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);

        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
        roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
                     mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);

        roce_set_field(mpt_entry->byte_64_buf_pa1,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
                       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
                       mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);

        mpt_entry->lkey = cpu_to_le32(mw->rkey);

        return 0;
}

static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
{
        return hns_roce_buf_offset(hr_cq->mtr.kmem,
                                   n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
}

static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
{
        struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);

        /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
        return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
                !!(n & hr_cq->cq_depth)) ? cqe : NULL;
}

static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci)
{
        *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M;
}

static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
                                   struct hns_roce_srq *srq)
{
        struct hns_roce_v2_cqe *cqe, *dest;
        u32 prod_index;
        int nfreed = 0;
        int wqe_index;
        u8 owner_bit;

        for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
             ++prod_index) {
                if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
                        break;
        }

        /*
         * Now backwards through the CQ, removing CQ entries
         * that match our QP by overwriting them with next entries.
         */
        while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
                cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
                if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
                                    V2_CQE_BYTE_16_LCL_QPN_S) &
                                    HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
                        if (srq &&
                            roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
                                wqe_index = roce_get_field(cqe->byte_4,
                                                     V2_CQE_BYTE_4_WQE_INDX_M,
                                                     V2_CQE_BYTE_4_WQE_INDX_S);
                                hns_roce_free_srq_wqe(srq, wqe_index);
                        }
                        ++nfreed;
                } else if (nfreed) {
                        dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
                                          hr_cq->ib_cq.cqe);
                        owner_bit = roce_get_bit(dest->byte_4,
                                                 V2_CQE_BYTE_4_OWNER_S);
                        memcpy(dest, cqe, sizeof(*cqe));
                        roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
                                     owner_bit);
                }
        }

        if (nfreed) {
                hr_cq->cons_index += nfreed;
                /*
                 * Make sure update of buffer contents is done before
                 * updating consumer index.
                 */
                wmb();
                hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
        }
}

static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
                                 struct hns_roce_srq *srq)
{
        spin_lock_irq(&hr_cq->lock);
        __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
        spin_unlock_irq(&hr_cq->lock);
}

static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
                                  struct hns_roce_cq *hr_cq, void *mb_buf,
                                  u64 *mtts, dma_addr_t dma_handle)
{
        struct hns_roce_v2_cq_context *cq_context;

        cq_context = mb_buf;
        memset(cq_context, 0, sizeof(*cq_context));

        roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
                       V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
        roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
                       V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
        roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
                       V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth));
        roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
                       V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);

        roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
                       V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);

        cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));

        roce_set_field(cq_context->byte_16_hop_addr,
                       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
                       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
                       upper_32_bits(to_hr_hw_page_addr(mtts[0])));
        roce_set_field(cq_context->byte_16_hop_addr,
                       V2_CQC_BYTE_16_CQE_HOP_NUM_M,
                       V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
                       HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);

        cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
        roce_set_field(cq_context->byte_24_pgsz_addr,
                       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
                       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
                       upper_32_bits(to_hr_hw_page_addr(mtts[1])));
        roce_set_field(cq_context->byte_24_pgsz_addr,
                       V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
                       V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
                       to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
        roce_set_field(cq_context->byte_24_pgsz_addr,
                       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
                       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
                       to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));

        cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);

        roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
                       V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));

        roce_set_bit(cq_context->byte_44_db_record,
                     V2_CQC_BYTE_44_DB_RECORD_EN_S,
                     (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0);

        roce_set_field(cq_context->byte_44_db_record,
                       V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
                       V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
                       ((u32)hr_cq->db.dma) >> 1);
        cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);

        roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
                       V2_CQC_BYTE_56_CQ_MAX_CNT_M,
                       V2_CQC_BYTE_56_CQ_MAX_CNT_S,
                       HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
        roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
                       V2_CQC_BYTE_56_CQ_PERIOD_M,
                       V2_CQC_BYTE_56_CQ_PERIOD_S,
                       HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
}

static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
                                     enum ib_cq_notify_flags flags)
{
        struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
        struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
        u32 notification_flag;
        __le32 doorbell[2];

        doorbell[0] = 0;
        doorbell[1] = 0;

        notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
                             V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
        /*
         * flags = 0; Notification Flag = 1, next
         * flags = 1; Notification Flag = 0, solocited
         */
        roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
                       hr_cq->cqn);
        roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
                       HNS_ROCE_V2_CQ_DB_NTR);
        roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
                       V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index);
        roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
                       V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
        roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
                     notification_flag);

        hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);

        return 0;
}

static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
                                                    struct hns_roce_qp **cur_qp,
                                                    struct ib_wc *wc)
{
        struct hns_roce_rinl_sge *sge_list;
        u32 wr_num, wr_cnt, sge_num;
        u32 sge_cnt, data_len, size;
        void *wqe_buf;

        wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
                                V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
        wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);

        sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
        sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
        wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt);
        data_len = wc->byte_len;

        for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
                size = min(sge_list[sge_cnt].len, data_len);
                memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);

                data_len -= size;
                wqe_buf += size;
        }

        if (unlikely(data_len)) {
                wc->status = IB_WC_LOC_LEN_ERR;
                return -EAGAIN;
        }

        return 0;
}

static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
                   int num_entries, struct ib_wc *wc)
{
        unsigned int left;
        int npolled = 0;

        left = wq->head - wq->tail;
        if (left == 0)
                return 0;

        left = min_t(unsigned int, (unsigned int)num_entries, left);
        while (npolled < left) {
                wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
                wc->status = IB_WC_WR_FLUSH_ERR;
                wc->vendor_err = 0;
                wc->qp = &hr_qp->ibqp;

                wq->tail++;
                wc++;
                npolled++;
        }

        return npolled;