/*
 * This file is part of the Chelsio FCoE driver for Linux.
 *
 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/compiler.h>
#include <linux/slab.h>
#include <asm/page.h>
#include <linux/cache.h>

#include "t4_values.h"
#include "csio_hw.h"
#include "csio_wr.h"
#include "csio_mb.h"
#include "csio_defs.h"

int csio_intr_coalesce_cnt;             /* value:SGE_INGRESS_RX_THRESHOLD[0] */
static int csio_sge_thresh_reg;         /* SGE_INGRESS_RX_THRESHOLD[0] */

int csio_intr_coalesce_time = 10;       /* value:SGE_TIMER_VALUE_1 */
static int csio_sge_timer_reg = 1;

#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val)                            \
        csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)

static void
csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
{
        sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +
                                                        reg * sizeof(uint32_t));
}

/* Free list buffer size */
static inline uint32_t
csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
{
        return sge->sge_fl_buf_size[buf->paddr & 0xF];
}

/* Size of the egress queue status page */
static inline uint32_t
csio_wr_qstat_pgsz(struct csio_hw *hw)
{
        return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ?  128 : 64;
}

/* Ring freelist doorbell */
static inline void
csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
{
        /*
         * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
         * number of bytes in the freelist queue. This translates to atleast
         * 8 freelist buffer pointers (since each pointer is 8 bytes).
         */
        if (flq->inc_idx >= 8) {
                csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |
                                  PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,
                                  MYPF_REG(SGE_PF_KDOORBELL_A));
                flq->inc_idx &= 7;
        }
}

/* Write a 0 cidx increment value to enable SGE interrupts for this queue */
static void
csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
{
        csio_wr_reg32(hw, CIDXINC_V(0)          |
                          INGRESSQID_V(iqid)    |
                          TIMERREG_V(X_TIMERREG_RESTART_COUNTER),
                          MYPF_REG(SGE_PF_GTS_A));
}

/*
 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
 * @hw: HW module.
 * @flq: Freelist queue.
 *
 * Fill up freelist buffer entries with buffers of size specified
 * in the size register.
 *
 */
static int
csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;
        __be64 *d = (__be64 *)(flq->vstart);
        struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
        uint64_t paddr;
        int sreg = flq->un.fl.sreg;
        int n = flq->credits;

        while (n--) {
                buf->len = sge->sge_fl_buf_size[sreg];
                buf->vaddr = dma_alloc_coherent(&hw->pdev->dev, buf->len,
                                                &buf->paddr, GFP_KERNEL);
                if (!buf->vaddr) {
                        csio_err(hw, "Could only fill %d buffers!\n", n + 1);
                        return -ENOMEM;
                }

                paddr = buf->paddr | (sreg & 0xF);

                *d++ = cpu_to_be64(paddr);
                buf++;
        }

        return 0;
}

/*
 * csio_wr_update_fl -
 * @hw: HW module.
 * @flq: Freelist queue.
 *
 *
 */
static inline void
csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
{

        flq->inc_idx += n;
        flq->pidx += n;
        if (unlikely(flq->pidx >= flq->credits))
                flq->pidx -= (uint16_t)flq->credits;

        CSIO_INC_STATS(flq, n_flq_refill);
}

/*
 * csio_wr_alloc_q - Allocate a WR queue and initialize it.
 * @hw: HW module
 * @qsize: Size of the queue in bytes
 * @wrsize: Since of WR in this queue, if fixed.
 * @type: Type of queue (Ingress/Egress/Freelist)
 * @owner: Module that owns this queue.
 * @nflb: Number of freelist buffers for FL.
 * @sreg: What is the FL buffer size register?
 * @iq_int_handler: Ingress queue handler in INTx mode.
 *
 * This function allocates and sets up a queue for the caller
 * of size qsize, aligned at the required boundary. This is subject to
 * be free entries being available in the queue array. If one is found,
 * it is initialized with the allocated queue, marked as being used (owner),
 * and a handle returned to the caller in form of the queue's index
 * into the q_arr array.
 * If user has indicated a freelist (by specifying nflb > 0), create
 * another queue (with its own index into q_arr) for the freelist. Allocate
 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
 * idx in the ingress queue's flq.idx. This is how a Freelist is associated
 * with its owning ingress queue.
 */
int
csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
                uint16_t type, void *owner, uint32_t nflb, int sreg,
                iq_handler_t iq_intx_handler)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_q   *q, *flq;
        int             free_idx = wrm->free_qidx;
        int             ret_idx = free_idx;
        uint32_t        qsz;
        int flq_idx;

        if (free_idx >= wrm->num_q) {
                csio_err(hw, "No more free queues.\n");
                return -1;
        }

        switch (type) {
        case CSIO_EGRESS:
                qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
                break;
        case CSIO_INGRESS:
                switch (wrsize) {
                case 16:
                case 32:
                case 64:
                case 128:
                        break;
                default:
                        csio_err(hw, "Invalid Ingress queue WR size:%d\n",
                                    wrsize);
                        return -1;
                }

                /*
                 * Number of elements must be a multiple of 16
                 * So this includes status page size
                 */
                qsz = ALIGN(qsize/wrsize, 16) * wrsize;

                break;
        case CSIO_FREELIST:
                qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
                break;
        default:
                csio_err(hw, "Invalid queue type: 0x%x\n", type);
                return -1;
        }

        q = wrm->q_arr[free_idx];

        q->vstart = dma_alloc_coherent(&hw->pdev->dev, qsz, &q->pstart,
                                       GFP_KERNEL);
        if (!q->vstart) {
                csio_err(hw,
                         "Failed to allocate DMA memory for "
                         "queue at id: %d size: %d\n", free_idx, qsize);
                return -1;
        }

        q->type         = type;
        q->owner        = owner;
        q->pidx         = q->cidx = q->inc_idx = 0;
        q->size         = qsz;
        q->wr_sz        = wrsize;       /* If using fixed size WRs */

        wrm->free_qidx++;

        if (type == CSIO_INGRESS) {
                /* Since queue area is set to zero */
                q->un.iq.genbit = 1;

                /*
                 * Ingress queue status page size is always the size of
                 * the ingress queue entry.
                 */
                q->credits      = (qsz - q->wr_sz) / q->wr_sz;
                q->vwrap        = (void *)((uintptr_t)(q->vstart) + qsz
                                                        - q->wr_sz);

                /* Allocate memory for FL if requested */
                if (nflb > 0) {
                        flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
                                                  sizeof(__be64), CSIO_FREELIST,
                                                  owner, 0, sreg, NULL);
                        if (flq_idx == -1) {
                                csio_err(hw,
                                         "Failed to allocate FL queue"
                                         " for IQ idx:%d\n", free_idx);
                                return -1;
                        }

                        /* Associate the new FL with the Ingress quue */
                        q->un.iq.flq_idx = flq_idx;

                        flq = wrm->q_arr[q->un.iq.flq_idx];
                        flq->un.fl.bufs = kcalloc(flq->credits,
                                                  sizeof(struct csio_dma_buf),
                                                  GFP_KERNEL);
                        if (!flq->un.fl.bufs) {
                                csio_err(hw,
                                         "Failed to allocate FL queue bufs"
                                         " for IQ idx:%d\n", free_idx);
                                return -1;
                        }

                        flq->un.fl.packen = 0;
                        flq->un.fl.offset = 0;
                        flq->un.fl.sreg = sreg;

                        /* Fill up the free list buffers */
                        if (csio_wr_fill_fl(hw, flq))
                                return -1;

                        /*
                         * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
                         * remains unpopulated,otherwise HW thinks
                         * FLQ is empty.
                         */
                        flq->pidx = flq->inc_idx = flq->credits - 8;
                } else {
                        q->un.iq.flq_idx = -1;
                }

                /* Associate the IQ INTx handler. */
                q->un.iq.iq_intx_handler = iq_intx_handler;

                csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;

        } else if (type == CSIO_EGRESS) {
                q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
                q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
                                                - csio_wr_qstat_pgsz(hw));
                csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
        } else { /* Freelist */
                q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
                q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
                                                - csio_wr_qstat_pgsz(hw));
                csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
        }

        return ret_idx;
}

/*
 * csio_wr_iq_create_rsp - Response handler for IQ creation.
 * @hw: The HW module.
 * @mbp: Mailbox.
 * @iq_idx: Ingress queue that got created.
 *
 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
 */
static int
csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
{
        struct csio_iq_params iqp;
        enum fw_retval retval;
        uint32_t iq_id;
        int flq_idx;

        memset(&iqp, 0, sizeof(struct csio_iq_params));

        csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);

        if (retval != FW_SUCCESS) {
                csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }

        csio_q_iqid(hw, iq_idx)         = iqp.iqid;
        csio_q_physiqid(hw, iq_idx)     = iqp.physiqid;
        csio_q_pidx(hw, iq_idx)         = csio_q_cidx(hw, iq_idx) = 0;
        csio_q_inc_idx(hw, iq_idx)      = 0;

        /* Actual iq-id. */
        iq_id = iqp.iqid - hw->wrm.fw_iq_start;

        /* Set the iq-id to iq map table. */
        if (iq_id >= CSIO_MAX_IQ) {
                csio_err(hw,
                         "Exceeding MAX_IQ(%d) supported!"
                         " iqid:%d rel_iqid:%d FW iq_start:%d\n",
                         CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }
        csio_q_set_intr_map(hw, iq_idx, iq_id);

        /*
         * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
         * ingress context of this queue. This will block interrupts to
         * this queue until the next GTS write. Therefore, we do a
         * 0-cidx increment GTS write for this queue just to clear the
         * interrupt_sent bit. This will re-enable interrupts to this
         * queue.
         */
        csio_wr_sge_intr_enable(hw, iqp.physiqid);

        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
        if (flq_idx != -1) {
                struct csio_q *flq = hw->wrm.q_arr[flq_idx];

                csio_q_flid(hw, flq_idx) = iqp.fl0id;
                csio_q_cidx(hw, flq_idx) = 0;
                csio_q_pidx(hw, flq_idx)    = csio_q_credits(hw, flq_idx) - 8;
                csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;

                /* Now update SGE about the buffers allocated during init */
                csio_wr_ring_fldb(hw, flq);
        }

        mempool_free(mbp, hw->mb_mempool);

        return 0;
}

/*
 * csio_wr_iq_create - Configure an Ingress queue with FW.
 * @hw: The HW module.
 * @priv: Private data object.
 * @iq_idx: Ingress queue index in the WR module.
 * @vec: MSIX vector.
 * @portid: PCIE Channel to be associated with this queue.
 * @async: Is this a FW asynchronous message handling queue?
 * @cbfn: Completion callback.
 *
 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
 * with alloc/write bits set.
 */
int
csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
                  uint32_t vec, uint8_t portid, bool async,
                  void (*cbfn) (struct csio_hw *, struct csio_mb *))
{
        struct csio_mb  *mbp;
        struct csio_iq_params iqp;
        int flq_idx;

        memset(&iqp, 0, sizeof(struct csio_iq_params));
        csio_q_portid(hw, iq_idx) = portid;

        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
        if (!mbp) {
                csio_err(hw, "IQ command out of memory!\n");
                return -ENOMEM;
        }

        switch (hw->intr_mode) {
        case CSIO_IM_INTX:
        case CSIO_IM_MSI:
                /* For interrupt forwarding queue only */
                if (hw->intr_iq_idx == iq_idx)
                        iqp.iqandst     = X_INTERRUPTDESTINATION_PCIE;
                else
                        iqp.iqandst     = X_INTERRUPTDESTINATION_IQ;
                iqp.iqandstindex        =
                        csio_q_physiqid(hw, hw->intr_iq_idx);
                break;
        case CSIO_IM_MSIX:
                iqp.iqandst             = X_INTERRUPTDESTINATION_PCIE;
                iqp.iqandstindex        = (uint16_t)vec;
                break;
        case CSIO_IM_NONE:
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }

        /* Pass in the ingress queue cmd parameters */
        iqp.pfn                 = hw->pfn;
        iqp.vfn                 = 0;
        iqp.iq_start            = 1;
        iqp.viid                = 0;
        iqp.type                = FW_IQ_TYPE_FL_INT_CAP;
        iqp.iqasynch            = async;
        if (csio_intr_coalesce_cnt)
                iqp.iqanus      = X_UPDATESCHEDULING_COUNTER_OPTTIMER;
        else
                iqp.iqanus      = X_UPDATESCHEDULING_TIMER;
        iqp.iqanud              = X_UPDATEDELIVERY_INTERRUPT;
        iqp.iqpciech            = portid;
        iqp.iqintcntthresh      = (uint8_t)csio_sge_thresh_reg;

        switch (csio_q_wr_sz(hw, iq_idx)) {
        case 16:
                iqp.iqesize = 0; break;
        case 32:
                iqp.iqesize = 1; break;
        case 64:
                iqp.iqesize = 2; break;
        case 128:
                iqp.iqesize = 3; break;
        }

        iqp.iqsize              = csio_q_size(hw, iq_idx) /
                                                csio_q_wr_sz(hw, iq_idx);
        iqp.iqaddr              = csio_q_pstart(hw, iq_idx);

        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
        if (flq_idx != -1) {
                enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id);
                struct csio_q *flq = hw->wrm.q_arr[flq_idx];

                iqp.fl0paden    = 1;
                iqp.fl0packen   = flq->un.fl.packen ? 1 : 0;
                iqp.fl0fbmin    = X_FETCHBURSTMIN_64B;
                iqp.fl0fbmax    = ((chip == CHELSIO_T5) ?
                                  X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B);
                iqp.fl0size     = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
                iqp.fl0addr     = csio_q_pstart(hw, flq_idx);
        }

        csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);

        if (csio_mb_issue(hw, mbp)) {
                csio_err(hw, "Issue of IQ cmd failed!\n");
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }

        if (cbfn != NULL)
                return 0;

        return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
}

/*
 * csio_wr_eq_create_rsp - Response handler for EQ creation.
 * @hw: The HW module.
 * @mbp: Mailbox.
 * @eq_idx: Egress queue that got created.
 *
 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
 */
static int
csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
{
        struct csio_eq_params eqp;
        enum fw_retval retval;

        memset(&eqp, 0, sizeof(struct csio_eq_params));

        csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);

        if (retval != FW_SUCCESS) {
                csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }

        csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid;
        csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
        csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0;
        csio_q_inc_idx(hw, eq_idx) = 0;

        mempool_free(mbp, hw->mb_mempool);

        return 0;
}

/*
 * csio_wr_eq_create - Configure an Egress queue with FW.
 * @hw: HW module.
 * @priv: Private data.
 * @eq_idx: Egress queue index in the WR module.
 * @iq_idx: Associated ingress queue index.
 * @cbfn: Completion callback.
 *
 * This API configures a offload egress queue with FW by issuing a
 * FW_EQ_OFLD_CMD  (with alloc + write ) mailbox.
 */
int
csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
                  int iq_idx, uint8_t portid,
                  void (*cbfn) (struct csio_hw *, struct csio_mb *))
{
        struct csio_mb  *mbp;
        struct csio_eq_params eqp;

        memset(&eqp, 0, sizeof(struct csio_eq_params));

        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
        if (!mbp) {
                csio_err(hw, "EQ command out of memory!\n");
                return -ENOMEM;
        }

        eqp.pfn                 = hw->pfn;
        eqp.vfn                 = 0;
        eqp.eqstart             = 1;
        eqp.hostfcmode          = X_HOSTFCMODE_STATUS_PAGE;
        eqp.iqid                = csio_q_iqid(hw, iq_idx);
        eqp.fbmin               = X_FETCHBURSTMIN_64B;
        eqp.fbmax               = X_FETCHBURSTMAX_512B;
        eqp.cidxfthresh         = 0;
        eqp.pciechn             = portid;
        eqp.eqsize              = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
        eqp.eqaddr              = csio_q_pstart(hw, eq_idx);

        csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
                                    &eqp, cbfn);

        if (csio_mb_issue(hw, mbp)) {
                csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
                mempool_free(mbp, hw->mb_mempool);
                return -EINVAL;
        }

        if (cbfn != NULL)
                return 0;

        return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
}

/*
 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
 * @hw: The HW module.
 * @mbp: Mailbox.
 * @iq_idx: Ingress queue that was freed.
 *
 * Handle FW_IQ_CMD (free) mailbox completion.
 */
static int
csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
{
        enum fw_retval retval = csio_mb_fw_retval(mbp);
        int rv = 0;

        if (retval != FW_SUCCESS)
                rv = -EINVAL;

        mempool_free(mbp, hw->mb_mempool);

        return rv;
}

/*
 * csio_wr_iq_destroy - Free an ingress queue.
 * @hw: The HW module.
 * @priv: Private data object.
 * @iq_idx: Ingress queue index to destroy
 * @cbfn: Completion callback.
 *
 * This API frees an ingress queue by issuing the FW_IQ_CMD
 * with the free bit set.
 */
static int
csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
                   void (*cbfn)(struct csio_hw *, struct csio_mb *))
{
        int rv = 0;
        struct csio_mb  *mbp;
        struct csio_iq_params iqp;
        int flq_idx;

        memset(&iqp, 0, sizeof(struct csio_iq_params));

        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
        if (!mbp)
                return -ENOMEM;

        iqp.pfn         = hw->pfn;
        iqp.vfn         = 0;
        iqp.iqid        = csio_q_iqid(hw, iq_idx);
        iqp.type        = FW_IQ_TYPE_FL_INT_CAP;

        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
        if (flq_idx != -1)
                iqp.fl0id = csio_q_flid(hw, flq_idx);
        else
                iqp.fl0id = 0xFFFF;

        iqp.fl1id = 0xFFFF;

        csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);

        rv = csio_mb_issue(hw, mbp);
        if (rv != 0) {
                mempool_free(mbp, hw->mb_mempool);
                return rv;
        }

        if (cbfn != NULL)
                return 0;

        return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
}

/*
 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
 * @hw: The HW module.
 * @mbp: Mailbox.
 * @eq_idx: Egress queue that was freed.
 *
 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
 */
static int
csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
{
        enum fw_retval retval = csio_mb_fw_retval(mbp);
        int rv = 0;

        if (retval != FW_SUCCESS)
                rv = -EINVAL;

        mempool_free(mbp, hw->mb_mempool);

        return rv;
}

/*
 * csio_wr_eq_destroy - Free an Egress queue.
 * @hw: The HW module.
 * @priv: Private data object.
 * @eq_idx: Egress queue index to destroy
 * @cbfn: Completion callback.
 *
 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
 * with the free bit set.
 */
static int
csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
                   void (*cbfn) (struct csio_hw *, struct csio_mb *))
{
        int rv = 0;
        struct csio_mb  *mbp;
        struct csio_eq_params eqp;

        memset(&eqp, 0, sizeof(struct csio_eq_params));

        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
        if (!mbp)
                return -ENOMEM;

        eqp.pfn         = hw->pfn;
        eqp.vfn         = 0;
        eqp.eqid        = csio_q_eqid(hw, eq_idx);

        csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);

        rv = csio_mb_issue(hw, mbp);
        if (rv != 0) {
                mempool_free(mbp, hw->mb_mempool);
                return rv;
        }

        if (cbfn != NULL)
                return 0;

        return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
}

/*
 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
 * @hw: HW module
 * @qidx: Egress queue index
 *
 * Cleanup the Egress queue status page.
 */
static void
csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
{
        struct csio_q   *q = csio_hw_to_wrm(hw)->q_arr[qidx];
        struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;

        memset(stp, 0, sizeof(*stp));
}

/*
 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
 * @hw: HW module
 * @qidx: Ingress queue index
 *
 * Cleanup the footer entries in the given ingress queue,
 * set to 1 the internal copy of genbit.
 */
static void
csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
{
        struct csio_wrm *wrm    = csio_hw_to_wrm(hw);
        struct csio_q   *q      = wrm->q_arr[qidx];
        void *wr;
        struct csio_iqwr_footer *ftr;
        uint32_t i = 0;

        /* set to 1 since we are just about zero out genbit */
        q->un.iq.genbit = 1;

        for (i = 0; i < q->credits; i++) {
                /* Get the WR */
                wr = (void *)((uintptr_t)q->vstart +
                                           (i * q->wr_sz));
                /* Get the footer */
                ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
                                          (q->wr_sz - sizeof(*ftr)));
                /* Zero out footer */
                memset(ftr, 0, sizeof(*ftr));
        }
}

int
csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
{
        int i, flq_idx;
        struct csio_q *q;
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        int rv;

        for (i = 0; i < wrm->free_qidx; i++) {
                q = wrm->q_arr[i];

                switch (q->type) {
                case CSIO_EGRESS:
                        if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
                                csio_wr_cleanup_eq_stpg(hw, i);
                                if (!cmd) {
                                        csio_q_eqid(hw, i) = CSIO_MAX_QID;
                                        continue;
                                }

                                rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
                                if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
                                        cmd = false;

                                csio_q_eqid(hw, i) = CSIO_MAX_QID;
                        }
                        fallthrough;
                case CSIO_INGRESS:
                        if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
                                csio_wr_cleanup_iq_ftr(hw, i);
                                if (!cmd) {
                                        csio_q_iqid(hw, i) = CSIO_MAX_QID;
                                        flq_idx = csio_q_iq_flq_idx(hw, i);
                                        if (flq_idx != -1)
                                                csio_q_flid(hw, flq_idx) =
                                                                CSIO_MAX_QID;
                                        continue;
                                }

                                rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
                                if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
                                        cmd = false;

                                csio_q_iqid(hw, i) = CSIO_MAX_QID;
                                flq_idx = csio_q_iq_flq_idx(hw, i);
                                if (flq_idx != -1)
                                        csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
                        }
                        break;
                default:
                        break;
                }
        }

        hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;

        return 0;
}

/*
 * csio_wr_get - Get requested size of WR entry/entries from queue.
 * @hw: HW module.
 * @qidx: Index of queue.
 * @size: Cumulative size of Work request(s).
 * @wrp: Work request pair.
 *
 * If requested credits are available, return the start address of the
 * work request in the work request pair. Set pidx accordingly and
 * return.
 *
 * NOTE about WR pair:
 * ==================
 * A WR can start towards the end of a queue, and then continue at the
 * beginning, since the queue is considered to be circular. This will
 * require a pair of address/size to be passed back to the caller -
 * hence Work request pair format.
 */
int
csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
            struct csio_wr_pair *wrp)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_q *q = wrm->q_arr[qidx];
        void *cwr = (void *)((uintptr_t)(q->vstart) +
                                                (q->pidx * CSIO_QCREDIT_SZ));
        struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
        uint16_t cidx = q->cidx = ntohs(stp->cidx);
        uint16_t pidx = q->pidx;
        uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ);
        int req_credits = req_sz / CSIO_QCREDIT_SZ;
        int credits;

        CSIO_DB_ASSERT(q->owner != NULL);
        CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
        CSIO_DB_ASSERT(cidx <= q->credits);

        /* Calculate credits */
        if (pidx > cidx) {
                credits = q->credits - (pidx - cidx) - 1;
        } else if (cidx > pidx) {
                credits = cidx - pidx - 1;
        } else {
                /* cidx == pidx, empty queue */
                credits = q->credits;
                CSIO_INC_STATS(q, n_qempty);
        }

        /*
         * Check if we have enough credits.
         * credits = 1 implies queue is full.
         */
        if (!credits || (req_credits > credits)) {
                CSIO_INC_STATS(q, n_qfull);
                return -EBUSY;
        }

        /*
         * If we are here, we have enough credits to satisfy the
         * request. Check if we are near the end of q, and if WR spills over.
         * If it does, use the first addr/size to cover the queue until
         * the end. Fit the remainder portion of the request at the top
         * of queue and return it in the second addr/len. Set pidx
         * accordingly.
         */
        if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
                wrp->addr1 = cwr;
                wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
                wrp->addr2 = q->vstart;
                wrp->size2 = req_sz - wrp->size1;
                q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
                                                        CSIO_QCREDIT_SZ);
                CSIO_INC_STATS(q, n_qwrap);
                CSIO_INC_STATS(q, n_eq_wr_split);
        } else {
                wrp->addr1 = cwr;
                wrp->size1 = req_sz;
                wrp->addr2 = NULL;
                wrp->size2 = 0;
                q->pidx += (uint16_t)req_credits;

                /* We are the end of queue, roll back pidx to top of queue */
                if (unlikely(q->pidx == q->credits)) {
                        q->pidx = 0;
                        CSIO_INC_STATS(q, n_qwrap);
                }
        }

        q->inc_idx = (uint16_t)req_credits;

        CSIO_INC_STATS(q, n_tot_reqs);

        return 0;
}

/*
 * csio_wr_copy_to_wrp - Copies given data into WR.
 * @data_buf - Data buffer
 * @wrp - Work request pair.
 * @wr_off - Work request offset.
 * @data_len - Data length.
 *
 * Copies the given data in Work Request. Work request pair(wrp) specifies
 * address information of Work request.
 * Returns: none
 */
void
csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
                   uint32_t wr_off, uint32_t data_len)
{
        uint32_t nbytes;

        /* Number of space available in buffer addr1 of WRP */
        nbytes = ((wrp->size1 - wr_off) >= data_len) ?
                                        data_len : (wrp->size1 - wr_off);

        memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
        data_len -= nbytes;

        /* Write the remaining data from the begining of circular buffer */
        if (data_len) {
                CSIO_DB_ASSERT(data_len <= wrp->size2);
                CSIO_DB_ASSERT(wrp->addr2 != NULL);
                memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
        }
}

/*
 * csio_wr_issue - Notify chip of Work request.
 * @hw: HW module.
 * @qidx: Index of queue.
 * @prio: 0: Low priority, 1: High priority
 *
 * Rings the SGE Doorbell by writing the current producer index of the passed
 * in queue into the register.
 *
 */
int
csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_q *q = wrm->q_arr[qidx];

        CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));

        wmb();
        /* Ring SGE Doorbell writing q->pidx into it */
        csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |
                          PIDX_T5_V(q->inc_idx) | DBTYPE_F,
                          MYPF_REG(SGE_PF_KDOORBELL_A));
        q->inc_idx = 0;

        return 0;
}

static inline uint32_t
csio_wr_avail_qcredits(struct csio_q *q)

{
        if (q->pidx > q->cidx)
                return q->pidx - q->cidx;
        else if (q->cidx > q->pidx)
                return q->credits - (q->cidx - q->pidx);
        else
                return 0;       /* cidx == pidx, empty queue */
}

/*
 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
 * @hw: HW module.
 * @flq: The freelist queue.
 *
 * Invalidate the driver's version of a freelist buffer entry,
 * without freeing the associated the DMA memory. The entry
 * to be invalidated is picked up from the current Free list
 * queue cidx.
 *
 */
static inline void
csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
{
        flq->cidx++;
        if (flq->cidx == flq->credits) {
                flq->cidx = 0;
                CSIO_INC_STATS(flq, n_qwrap);
        }
}

/*
 * csio_wr_process_fl - Process a freelist completion.
 * @hw: HW module.
 * @q: The ingress queue attached to the Freelist.
 * @wr: The freelist completion WR in the ingress queue.
 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
 * @iq_handler: Caller's handler for this completion.
 * @priv: Private pointer of caller
 *
 */
static inline void
csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
                   void *wr, uint32_t len_to_qid,
                   void (*iq_handler)(struct csio_hw *, void *,
                                      uint32_t, struct csio_fl_dma_buf *,
                                      void *),
                   void *priv)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;
        struct csio_fl_dma_buf flb;
        struct csio_dma_buf *buf, *fbuf;
        uint32_t bufsz, len, lastlen = 0;
        int n;
        struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];

        CSIO_DB_ASSERT(flq != NULL);

        len = len_to_qid;

        if (len & IQWRF_NEWBUF) {
                if (flq->un.fl.offset > 0) {
                        csio_wr_inval_flq_buf(hw, flq);
                        flq->un.fl.offset = 0;
                }
                len = IQWRF_LEN_GET(len);
        }

        CSIO_DB_ASSERT(len != 0);

        flb.totlen = len;

        /* Consume all freelist buffers used for len bytes */
        for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
                buf = &flq->un.fl.bufs[flq->cidx];
                bufsz = csio_wr_fl_bufsz(sge, buf);

                fbuf->paddr     = buf->paddr;
                fbuf->vaddr     = buf->vaddr;

                flb.offset      = flq->un.fl.offset;
                lastlen         = min(bufsz, len);
                fbuf->len       = lastlen;

                len -= lastlen;
                if (!len)
                        break;
                csio_wr_inval_flq_buf(hw, flq);
        }

        flb.defer_free = flq->un.fl.packen ? 0 : 1;

        iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
                   &flb, priv);

        if (flq->un.fl.packen)
                flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
        else
                csio_wr_inval_flq_buf(hw, flq);

}

/*
 * csio_is_new_iqwr - Is this a new Ingress queue entry ?
 * @q: Ingress quueue.
 * @ftr: Ingress queue WR SGE footer.
 *
 * The entry is new if our generation bit matches the corresponding
 * bit in the footer of the current WR.
 */
static inline bool
csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
{
        return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
}

/*
 * csio_wr_process_iq - Process elements in Ingress queue.
 * @hw:  HW pointer
 * @qidx: Index of queue
 * @iq_handler: Handler for this queue
 * @priv: Caller's private pointer
 *
 * This routine walks through every entry of the ingress queue, calling
 * the provided iq_handler with the entry, until the generation bit
 * flips.
 */
int
csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
                   void (*iq_handler)(struct csio_hw *, void *,
                                      uint32_t, struct csio_fl_dma_buf *,
                                      void *),
                   void *priv)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
        struct csio_iqwr_footer *ftr;
        uint32_t wr_type, fw_qid, qid;
        struct csio_q *q_completed;
        struct csio_q *flq = csio_iq_has_fl(q) ?
                                        wrm->q_arr[q->un.iq.flq_idx] : NULL;
        int rv = 0;

        /* Get the footer */
        ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
                                          (q->wr_sz - sizeof(*ftr)));

        /*
         * When q wrapped around last time, driver should have inverted
         * ic.genbit as well.
         */
        while (csio_is_new_iqwr(q, ftr)) {

                CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
                                                (uintptr_t)q->vwrap);
                rmb();
                wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);

                switch (wr_type) {
                case X_RSPD_TYPE_CPL:
                        /* Subtract footer from WR len */
                        iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
                        break;
                case X_RSPD_TYPE_FLBUF:
                        csio_wr_process_fl(hw, q, wr,
                                           ntohl(ftr->pldbuflen_qid),
                                           iq_handler, priv);
                        break;
                case X_RSPD_TYPE_INTR:
                        fw_qid = ntohl(ftr->pldbuflen_qid);
                        qid = fw_qid - wrm->fw_iq_start;
                        q_completed = hw->wrm.intr_map[qid];

                        if (unlikely(qid ==
                                        csio_q_physiqid(hw, hw->intr_iq_idx))) {
                                /*
                                 * We are already in the Forward Interrupt
                                 * Interrupt Queue Service! Do-not service
                                 * again!
                                 *
                                 */
                        } else {
                                CSIO_DB_ASSERT(q_completed);
                                CSIO_DB_ASSERT(
                                        q_completed->un.iq.iq_intx_handler);

                                /* Call the queue handler. */
                                q_completed->un.iq.iq_intx_handler(hw, NULL,
                                                0, NULL, (void *)q_completed);
                        }
                        break;
                default:
                        csio_warn(hw, "Unknown resp type 0x%x received\n",
                                 wr_type);
                        CSIO_INC_STATS(q, n_rsp_unknown);
                        break;
                }

                /*
                 * Ingress *always* has fixed size WR entries. Therefore,
                 * there should always be complete WRs towards the end of
                 * queue.
                 */
                if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {

                        /* Roll over to start of queue */
                        q->cidx = 0;
                        wr      = q->vstart;

                        /* Toggle genbit */
                        q->un.iq.genbit ^= 0x1;

                        CSIO_INC_STATS(q, n_qwrap);
                } else {
                        q->cidx++;
                        wr      = (void *)((uintptr_t)(q->vstart) +
                                           (q->cidx * q->wr_sz));
                }

                ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
                                                  (q->wr_sz - sizeof(*ftr)));
                q->inc_idx++;

        } /* while (q->un.iq.genbit == hdr->genbit) */

        /*
         * We need to re-arm SGE interrupts in case we got a stray interrupt,
         * especially in msix mode. With INTx, this may be a common occurence.
         */
        if (unlikely(!q->inc_idx)) {
                CSIO_INC_STATS(q, n_stray_comp);
                rv = -EINVAL;
                goto restart;
        }

        /* Replenish free list buffers if pending falls below low water mark */
        if (flq) {
                uint32_t avail  = csio_wr_avail_qcredits(flq);
                if (avail <= 16) {
                        /* Make sure in FLQ, atleast 1 credit (8 FL buffers)
                         * remains unpopulated otherwise HW thinks
                         * FLQ is empty.
                         */
                        csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
                        csio_wr_ring_fldb(hw, flq);
                }
        }

restart:
        /* Now inform SGE about our incremental index value */
        csio_wr_reg32(hw, CIDXINC_V(q->inc_idx)         |
                          INGRESSQID_V(q->un.iq.physiqid)       |
                          TIMERREG_V(csio_sge_timer_reg),
                          MYPF_REG(SGE_PF_GTS_A));
        q->stats.n_tot_rsps += q->inc_idx;

        q->inc_idx = 0;

        return rv;
}

int
csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
                   void (*iq_handler)(struct csio_hw *, void *,
                                      uint32_t, struct csio_fl_dma_buf *,
                                      void *),
                   void *priv)
{
        struct csio_wrm *wrm    = csio_hw_to_wrm(hw);
        struct csio_q   *iq     = wrm->q_arr[qidx];

        return csio_wr_process_iq(hw, iq, iq_handler, priv);
}

static int
csio_closest_timer(struct csio_sge *s, int time)
{
        int i, delta, match = 0, min_delta = INT_MAX;

        for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
                delta = time - s->timer_val[i];
                if (delta < 0)
                        delta = -delta;
                if (delta < min_delta) {
                        min_delta = delta;
                        match = i;
                }
        }
        return match;
}

static int
csio_closest_thresh(struct csio_sge *s, int cnt)
{
        int i, delta, match = 0, min_delta = INT_MAX;

        for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
                delta = cnt - s->counter_val[i];
                if (delta < 0)
                        delta = -delta;
                if (delta < min_delta) {
                        min_delta = delta;
                        match = i;
                }
        }
        return match;
}

static void
csio_wr_fixup_host_params(struct csio_hw *hw)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;
        uint32_t clsz = L1_CACHE_BYTES;
        uint32_t s_hps = PAGE_SHIFT - 10;
        uint32_t stat_len = clsz > 64 ? 128 : 64;
        u32 fl_align = clsz < 32 ? 32 : clsz;
        u32 pack_align;
        u32 ingpad, ingpack;

        csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |
                      HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |
                      HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |
                      HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),
                      SGE_HOST_PAGE_SIZE_A);

        /* T5 introduced the separation of the Free List Padding and
         * Packing Boundaries.  Thus, we can select a smaller Padding
         * Boundary to avoid uselessly chewing up PCIe Link and Memory
         * Bandwidth, and use a Packing Boundary which is large enough
         * to avoid false sharing between CPUs, etc.
         *
         * For the PCI Link, the smaller the Padding Boundary the
         * better.  For the Memory Controller, a smaller Padding
         * Boundary is better until we cross under the Memory Line
         * Size (the minimum unit of transfer to/from Memory).  If we
         * have a Padding Boundary which is smaller than the Memory
         * Line Size, that'll involve a Read-Modify-Write cycle on the
         * Memory Controller which is never good.
         */

        /* We want the Packing Boundary to be based on the Cache Line
         * Size in order to help avoid False Sharing performance
         * issues between CPUs, etc.  We also want the Packing
         * Boundary to incorporate the PCI-E Maximum Payload Size.  We
         * get best performance when the Packing Boundary is a
         * multiple of the Maximum Payload Size.
         */
        pack_align = fl_align;
        if (pci_is_pcie(hw->pdev)) {
                u32 mps, mps_log;
                u16 devctl;

                /* The PCIe Device Control Maximum Payload Size field
                 * [bits 7:5] encodes sizes as powers of 2 starting at
                 * 128 bytes.
                 */
                pcie_capability_read_word(hw->pdev, PCI_EXP_DEVCTL, &devctl);
                mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;
                mps = 1 << mps_log;
                if (mps > pack_align)
                        pack_align = mps;
        }

        /* T5/T6 have a special interpretation of the "0"
         * value for the Packing Boundary.  This corresponds to 16
         * bytes instead of the expected 32 bytes.
         */
        if (pack_align <= 16) {
                ingpack = INGPACKBOUNDARY_16B_X;
                fl_align = 16;
        } else if (pack_align == 32) {
                ingpack = INGPACKBOUNDARY_64B_X;
                fl_align = 64;
        } else {
                u32 pack_align_log = fls(pack_align) - 1;

                ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;
                fl_align = pack_align;
        }

        /* Use the smallest Ingress Padding which isn't smaller than
         * the Memory Controller Read/Write Size.  We'll take that as
         * being 8 bytes since we don't know of any system with a
         * wider Memory Controller Bus Width.
         */
        if (csio_is_t5(hw->pdev->device & CSIO_HW_CHIP_MASK))
                ingpad = INGPADBOUNDARY_32B_X;
        else
                ingpad = T6_INGPADBOUNDARY_8B_X;

        csio_set_reg_field(hw, SGE_CONTROL_A,
                           INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
                           EGRSTATUSPAGESIZE_F,
                           INGPADBOUNDARY_V(ingpad) |
                           EGRSTATUSPAGESIZE_V(stat_len != 64));
        csio_set_reg_field(hw, SGE_CONTROL2_A,
                           INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),
                           INGPACKBOUNDARY_V(ingpack));

        /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
        csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);

        /*
         * If using hard params, the following will get set correctly
         * in csio_wr_set_sge().
         */
        if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {
                csio_wr_reg32(hw,
                        (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +
                        fl_align - 1) & ~(fl_align - 1),
                        SGE_FL_BUFFER_SIZE2_A);
                csio_wr_reg32(hw,
                        (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +
                        fl_align - 1) & ~(fl_align - 1),
                        SGE_FL_BUFFER_SIZE3_A);
        }

        sge->csio_fl_align = fl_align;

        csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);

        /* default value of rx_dma_offset of the NIC driver */
        csio_set_reg_field(hw, SGE_CONTROL_A,
                           PKTSHIFT_V(PKTSHIFT_M),
                           PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));

        csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,
                                    CSUM_HAS_PSEUDO_HDR_F, 0);
}

static void
csio_init_intr_coalesce_parms(struct csio_hw *hw)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;

        csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
        if (csio_intr_coalesce_cnt) {
                csio_sge_thresh_reg = 0;
                csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
                return;
        }

        csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
}

/*
 * csio_wr_get_sge - Get SGE register values.
 * @hw: HW module.
 *
 * Used by non-master functions and by master-functions relying on config file.
 */
static void
csio_wr_get_sge(struct csio_hw *hw)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;
        uint32_t ingpad;
        int i;
        u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
        u32 ingress_rx_threshold;

        sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);

        ingpad = INGPADBOUNDARY_G(sge->sge_control);

        switch (ingpad) {
        case X_INGPCIEBOUNDARY_32B:
                sge->csio_fl_align = 32; break;
        case X_INGPCIEBOUNDARY_64B:
                sge->csio_fl_align = 64; break;
        case X_INGPCIEBOUNDARY_128B:
                sge->csio_fl_align = 128; break;
        case X_INGPCIEBOUNDARY_256B:
                sge->csio_fl_align = 256; break;
        case X_INGPCIEBOUNDARY_512B:
                sge->csio_fl_align = 512; break;
        case X_INGPCIEBOUNDARY_1024B:
                sge->csio_fl_align = 1024; break;
        case X_INGPCIEBOUNDARY_2048B:
                sge->csio_fl_align = 2048; break;
        case X_INGPCIEBOUNDARY_4096B:
                sge->csio_fl_align = 4096; break;
        }

        for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
                csio_get_flbuf_size(hw, sge, i);

        timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);
        timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);
        timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);

        sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE0_G(timer_value_0_and_1));
        sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE1_G(timer_value_0_and_1));
        sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE2_G(timer_value_2_and_3));
        sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE3_G(timer_value_2_and_3));
        sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE4_G(timer_value_4_and_5));
        sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
                                        TIMERVALUE5_G(timer_value_4_and_5));

        ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);
        sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
        sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
        sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
        sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);

        csio_init_intr_coalesce_parms(hw);
}

/*
 * csio_wr_set_sge - Initialize SGE registers
 * @hw: HW module.
 *
 * Used by Master function to initialize SGE registers in the absence
 * of a config file.
 */
static void
csio_wr_set_sge(struct csio_hw *hw)
{
        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
        struct csio_sge *sge = &wrm->sge;
        int i;

        /*
         * Set up our basic SGE mode to deliver CPL messages to our Ingress
         * Queue and Packet Date to the Free List.
         */
        csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);

        sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);

        /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */

        /*
         * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
         * and generate an interrupt when this occurs so we can recover.
         */
        csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,
                           LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
                           LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
        csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,
                           HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
                           HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));

        csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,
                           ENABLE_DROP_F);

        /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */

        CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
        csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)
                      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);
        csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)
                      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);
        CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
        CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
        CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
        CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
        CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);

        for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
                csio_get_flbuf_size(hw, sge, i);

        /* Initialize interrupt coalescing attributes */
        sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
        sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
        sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
        sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
        sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
        sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;

        sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
        sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
        sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
        sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;

        csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |
                      THRESHOLD_1_V(sge->counter_val[1]) |
                      THRESHOLD_2_V(sge->counter_val[2]) |
                      THRESHOLD_3_V(sge->counter_val[3]),
                      SGE_INGRESS_RX_THRESHOLD_A);

        csio_wr_reg32(hw,
                   TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
                   TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),
                   SGE_TIMER_VALUE_0_AND_1_A);

        csio_wr_reg32(hw,
                   TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
                   TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),
                   SGE_TIMER_VALUE_2_AND_3_A);

        csio_wr_reg32(hw,
                   TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
                   TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),
                   SGE_TIMER_VALUE_4_AND_5_A);

        csio_init_intr_coalesce_parms(hw);
}

void
csio_wr_sge_init(struct csio_hw *hw)
{
        /*
         * If we are master and chip is not initialized:
         *    - If we plan to use the config file, we need to fixup some
         *      host specific registers, and read the rest of the SGE
         *      configuration.
         *    - If we dont plan to use the config file, we need to initialize
         *      SGE entirely, including fixing the host specific registers.
         * If we are master and chip is initialized, just read and work off of
         *      the already initialized SGE values.
         * If we arent the master, we are only allowed to read and work off of
         *      the already initialized SGE values.
         *
         * Therefore, before calling this function, we assume that the master-
         * ship of the card, state and whether to use config file or not, have
         * already been decided.
         */
        if (csio_is_hw_master(hw)) {
                if (hw->fw_state != CSIO_DEV_STATE_INIT)
                        csio_wr_fixup_host_params(hw);

                if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
                        csio_wr_get_sge(hw);
                else
                        csio_wr_set_sge(hw);
        } else
                csio_wr_get_sge(hw);
}

/*
 * csio_wrm_init - Initialize Work request module.
 * @wrm: WR module
 * @hw: HW pointer
 *
 * Allocates memory for an array of queue pointers starting at q_arr.
 */
int
csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
{
        int i;

        if (!wrm->num_q) {
                csio_err(hw, "Num queues is not set\n");
                return -EINVAL;
        }

        wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL);
        if (!wrm->q_arr)
                goto err;

        for (i = 0; i < wrm->num_q; i++) {
                wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
                if (!wrm->q_arr[i]) {
                        while (--i >= 0)
                                kfree(wrm->q_arr[i]);
                        goto err_free_arr;
                }
        }
        wrm->free_qidx  = 0;

        return 0;

err_free_arr:
        kfree(wrm->q_arr);
err:
        return -ENOMEM;
}

/*
 * csio_wrm_exit - Initialize Work request module.
 * @wrm: WR module
 * @hw: HW module
 *
 * Uninitialize WR module. Free q_arr and pointers in it.
 * We have the additional job of freeing the DMA memory associated
 * with the queues.
 */
void
csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
{
        int i;
        uint32_t j;
        struct csio_q *q;
        struct csio_dma_buf *buf;

        for (i = 0; i < wrm->num_q; i++) {
                q = wrm->q_arr[i];

                if (wrm->free_qidx && (i < wrm->free_qidx)) {
                        if (q->type == CSIO_FREELIST) {
                                if (!q->un.fl.bufs)
                                        continue;
                                for (j = 0; j < q->credits; j++) {
                                        buf = &q->un.fl.bufs[j];
                                        if (!buf->vaddr)
                                                continue;
                                        dma_free_coherent(&hw->pdev->dev,
                                                        buf->len, buf->vaddr,
                                                        buf->paddr);
                                }
                                kfree(q->un.fl.bufs);
                        }
                        dma_free_coherent(&hw->pdev->dev, q->size,
                                        q->vstart, q->pstart);
                }
                kfree(q);
        }

        hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;

        kfree(wrm->q_arr);
}