/*******************************************************************
 * This file is part of the Emulex Linux Device Driver for         *
 * Fibre Channel Host Bus Adapters.                                *
 * Copyright (C) 2004-2009 Emulex.  All rights reserved.           *
 * EMULEX and SLI are trademarks of Emulex.                        *
 * www.emulex.com                                                  *
 * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
 *                                                                 *
 * This program is free software; you can redistribute it and/or   *
 * modify it under the terms of version 2 of the GNU General       *
 * Public License as published by the Free Software Foundation.    *
 * This program is distributed in the hope that it will be useful. *
 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
 * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
 * more details, a copy of which can be found in the file COPYING  *
 * included with this package.                                     *
 *******************************************************************/

#include <linux/blkdev.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/delay.h>

#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport_fc.h>
#include <scsi/fc/fc_fs.h>

#include "lpfc_hw4.h"
#include "lpfc_hw.h"
#include "lpfc_sli.h"
#include "lpfc_sli4.h"
#include "lpfc_nl.h"
#include "lpfc_disc.h"
#include "lpfc_scsi.h"
#include "lpfc.h"
#include "lpfc_crtn.h"
#include "lpfc_logmsg.h"
#include "lpfc_compat.h"
#include "lpfc_debugfs.h"
#include "lpfc_vport.h"

/* There are only four IOCB completion types. */
typedef enum _lpfc_iocb_type {
        LPFC_UNKNOWN_IOCB,
        LPFC_UNSOL_IOCB,
        LPFC_SOL_IOCB,
        LPFC_ABORT_IOCB
} lpfc_iocb_type;


/* Provide function prototypes local to this module. */
static int lpfc_sli_issue_mbox_s4(struct lpfc_hba *, LPFC_MBOXQ_t *,
                                  uint32_t);
static int lpfc_sli4_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *,
                            uint8_t *, uint32_t *);

static IOCB_t *
lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
{
        return &iocbq->iocb;
}

/**
 * lpfc_sli4_wq_put - Put a Work Queue Entry on an Work Queue
 * @q: The Work Queue to operate on.
 * @wqe: The work Queue Entry to put on the Work queue.
 *
 * This routine will copy the contents of @wqe to the next available entry on
 * the @q. This function will then ring the Work Queue Doorbell to signal the
 * HBA to start processing the Work Queue Entry. This function returns 0 if
 * successful. If no entries are available on @q then this function will return
 * -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
static uint32_t
lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
{
        union lpfc_wqe *temp_wqe = q->qe[q->host_index].wqe;
        struct lpfc_register doorbell;
        uint32_t host_index;

        /* If the host has not yet processed the next entry then we are done */
        if (((q->host_index + 1) % q->entry_count) == q->hba_index)
                return -ENOMEM;
        /* set consumption flag every once in a while */
        if (!((q->host_index + 1) % LPFC_RELEASE_NOTIFICATION_INTERVAL))
                bf_set(lpfc_wqe_gen_wqec, &wqe->generic, 1);

        lpfc_sli_pcimem_bcopy(wqe, temp_wqe, q->entry_size);

        /* Update the host index before invoking device */
        host_index = q->host_index;
        q->host_index = ((q->host_index + 1) % q->entry_count);

        /* Ring Doorbell */
        doorbell.word0 = 0;
        bf_set(lpfc_wq_doorbell_num_posted, &doorbell, 1);
        bf_set(lpfc_wq_doorbell_index, &doorbell, host_index);
        bf_set(lpfc_wq_doorbell_id, &doorbell, q->queue_id);
        writel(doorbell.word0, q->phba->sli4_hba.WQDBregaddr);
        readl(q->phba->sli4_hba.WQDBregaddr); /* Flush */

        return 0;
}

/**
 * lpfc_sli4_wq_release - Updates internal hba index for WQ
 * @q: The Work Queue to operate on.
 * @index: The index to advance the hba index to.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * Work Queue Entries by the HBA. When the HBA indicates that it has consumed
 * an entry the host calls this function to update the queue's internal
 * pointers. This routine returns the number of entries that were consumed by
 * the HBA.
 **/
static uint32_t
lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index)
{
        uint32_t released = 0;

        if (q->hba_index == index)
                return 0;
        do {
                q->hba_index = ((q->hba_index + 1) % q->entry_count);
                released++;
        } while (q->hba_index != index);
        return released;
}

/**
 * lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue
 * @q: The Mailbox Queue to operate on.
 * @wqe: The Mailbox Queue Entry to put on the Work queue.
 *
 * This routine will copy the contents of @mqe to the next available entry on
 * the @q. This function will then ring the Work Queue Doorbell to signal the
 * HBA to start processing the Work Queue Entry. This function returns 0 if
 * successful. If no entries are available on @q then this function will return
 * -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
static uint32_t
lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
{
        struct lpfc_mqe *temp_mqe = q->qe[q->host_index].mqe;
        struct lpfc_register doorbell;
        uint32_t host_index;

        /* If the host has not yet processed the next entry then we are done */
        if (((q->host_index + 1) % q->entry_count) == q->hba_index)
                return -ENOMEM;
        lpfc_sli_pcimem_bcopy(mqe, temp_mqe, q->entry_size);
        /* Save off the mailbox pointer for completion */
        q->phba->mbox = (MAILBOX_t *)temp_mqe;

        /* Update the host index before invoking device */
        host_index = q->host_index;
        q->host_index = ((q->host_index + 1) % q->entry_count);

        /* Ring Doorbell */
        doorbell.word0 = 0;
        bf_set(lpfc_mq_doorbell_num_posted, &doorbell, 1);
        bf_set(lpfc_mq_doorbell_id, &doorbell, q->queue_id);
        writel(doorbell.word0, q->phba->sli4_hba.MQDBregaddr);
        readl(q->phba->sli4_hba.MQDBregaddr); /* Flush */
        return 0;
}

/**
 * lpfc_sli4_mq_release - Updates internal hba index for MQ
 * @q: The Mailbox Queue to operate on.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * a Mailbox Queue Entry by the HBA. When the HBA indicates that it has consumed
 * an entry the host calls this function to update the queue's internal
 * pointers. This routine returns the number of entries that were consumed by
 * the HBA.
 **/
static uint32_t
lpfc_sli4_mq_release(struct lpfc_queue *q)
{
        /* Clear the mailbox pointer for completion */
        q->phba->mbox = NULL;
        q->hba_index = ((q->hba_index + 1) % q->entry_count);
        return 1;
}

/**
 * lpfc_sli4_eq_get - Gets the next valid EQE from a EQ
 * @q: The Event Queue to get the first valid EQE from
 *
 * This routine will get the first valid Event Queue Entry from @q, update
 * the queue's internal hba index, and return the EQE. If no valid EQEs are in
 * the Queue (no more work to do), or the Queue is full of EQEs that have been
 * processed, but not popped back to the HBA then this routine will return NULL.
 **/
static struct lpfc_eqe *
lpfc_sli4_eq_get(struct lpfc_queue *q)
{
        struct lpfc_eqe *eqe = q->qe[q->hba_index].eqe;

        /* If the next EQE is not valid then we are done */
        if (!bf_get(lpfc_eqe_valid, eqe))
                return NULL;
        /* If the host has not yet processed the next entry then we are done */
        if (((q->hba_index + 1) % q->entry_count) == q->host_index)
                return NULL;

        q->hba_index = ((q->hba_index + 1) % q->entry_count);
        return eqe;
}

/**
 * lpfc_sli4_eq_release - Indicates the host has finished processing an EQ
 * @q: The Event Queue that the host has completed processing for.
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will mark all Event Queue Entries on @q, from the last
 * known completed entry to the last entry that was processed, as completed
 * by clearing the valid bit for each completion queue entry. Then it will
 * notify the HBA, by ringing the doorbell, that the EQEs have been processed.
 * The internal host index in the @q will be updated by this routine to indicate
 * that the host has finished processing the entries. The @arm parameter
 * indicates that the queue should be rearmed when ringing the doorbell.
 *
 * This function will return the number of EQEs that were popped.
 **/
uint32_t
lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm)
{
        uint32_t released = 0;
        struct lpfc_eqe *temp_eqe;
        struct lpfc_register doorbell;

        /* while there are valid entries */
        while (q->hba_index != q->host_index) {
                temp_eqe = q->qe[q->host_index].eqe;
                bf_set(lpfc_eqe_valid, temp_eqe, 0);
                released++;
                q->host_index = ((q->host_index + 1) % q->entry_count);
        }
        if (unlikely(released == 0 && !arm))
                return 0;

        /* ring doorbell for number popped */
        doorbell.word0 = 0;
        if (arm) {
                bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
                bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
        }
        bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released);
        bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
        bf_set(lpfc_eqcq_doorbell_eqid, &doorbell, q->queue_id);
        writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
        return released;
}

/**
 * lpfc_sli4_cq_get - Gets the next valid CQE from a CQ
 * @q: The Completion Queue to get the first valid CQE from
 *
 * This routine will get the first valid Completion Queue Entry from @q, update
 * the queue's internal hba index, and return the CQE. If no valid CQEs are in
 * the Queue (no more work to do), or the Queue is full of CQEs that have been
 * processed, but not popped back to the HBA then this routine will return NULL.
 **/
static struct lpfc_cqe *
lpfc_sli4_cq_get(struct lpfc_queue *q)
{
        struct lpfc_cqe *cqe;

        /* If the next CQE is not valid then we are done */
        if (!bf_get(lpfc_cqe_valid, q->qe[q->hba_index].cqe))
                return NULL;
        /* If the host has not yet processed the next entry then we are done */
        if (((q->hba_index + 1) % q->entry_count) == q->host_index)
                return NULL;

        cqe = q->qe[q->hba_index].cqe;
        q->hba_index = ((q->hba_index + 1) % q->entry_count);
        return cqe;
}

/**
 * lpfc_sli4_cq_release - Indicates the host has finished processing a CQ
 * @q: The Completion Queue that the host has completed processing for.
 * @arm: Indicates whether the host wants to arms this CQ.
 *
 * This routine will mark all Completion queue entries on @q, from the last
 * known completed entry to the last entry that was processed, as completed
 * by clearing the valid bit for each completion queue entry. Then it will
 * notify the HBA, by ringing the doorbell, that the CQEs have been processed.
 * The internal host index in the @q will be updated by this routine to indicate
 * that the host has finished processing the entries. The @arm parameter
 * indicates that the queue should be rearmed when ringing the doorbell.
 *
 * This function will return the number of CQEs that were released.
 **/
uint32_t
lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm)
{
        uint32_t released = 0;
        struct lpfc_cqe *temp_qe;
        struct lpfc_register doorbell;

        /* while there are valid entries */
        while (q->hba_index != q->host_index) {
                temp_qe = q->qe[q->host_index].cqe;
                bf_set(lpfc_cqe_valid, temp_qe, 0);
                released++;
                q->host_index = ((q->host_index + 1) % q->entry_count);
        }
        if (unlikely(released == 0 && !arm))
                return 0;

        /* ring doorbell for number popped */
        doorbell.word0 = 0;
        if (arm)
                bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1);
        bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released);
        bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION);
        bf_set(lpfc_eqcq_doorbell_cqid, &doorbell, q->queue_id);
        writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
        return released;
}

/**
 * lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue
 * @q: The Header Receive Queue to operate on.
 * @wqe: The Receive Queue Entry to put on the Receive queue.
 *
 * This routine will copy the contents of @wqe to the next available entry on
 * the @q. This function will then ring the Receive Queue Doorbell to signal the
 * HBA to start processing the Receive Queue Entry. This function returns the
 * index that the rqe was copied to if successful. If no entries are available
 * on @q then this function will return -ENOMEM.
 * The caller is expected to hold the hbalock when calling this routine.
 **/
static int
lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
                 struct lpfc_rqe *hrqe, struct lpfc_rqe *drqe)
{
        struct lpfc_rqe *temp_hrqe = hq->qe[hq->host_index].rqe;
        struct lpfc_rqe *temp_drqe = dq->qe[dq->host_index].rqe;
        struct lpfc_register doorbell;
        int put_index = hq->host_index;

        if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)
                return -EINVAL;
        if (hq->host_index != dq->host_index)
                return -EINVAL;
        /* If the host has not yet processed the next entry then we are done */
        if (((hq->host_index + 1) % hq->entry_count) == hq->hba_index)
                return -EBUSY;
        lpfc_sli_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);
        lpfc_sli_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);

        /* Update the host index to point to the next slot */
        hq->host_index = ((hq->host_index + 1) % hq->entry_count);
        dq->host_index = ((dq->host_index + 1) % dq->entry_count);

        /* Ring The Header Receive Queue Doorbell */
        if (!(hq->host_index % LPFC_RQ_POST_BATCH)) {
                doorbell.word0 = 0;
                bf_set(lpfc_rq_doorbell_num_posted, &doorbell,
                       LPFC_RQ_POST_BATCH);
                bf_set(lpfc_rq_doorbell_id, &doorbell, hq->queue_id);
                writel(doorbell.word0, hq->phba->sli4_hba.RQDBregaddr);
        }
        return put_index;
}

/**
 * lpfc_sli4_rq_release - Updates internal hba index for RQ
 * @q: The Header Receive Queue to operate on.
 *
 * This routine will update the HBA index of a queue to reflect consumption of
 * one Receive Queue Entry by the HBA. When the HBA indicates that it has
 * consumed an entry the host calls this function to update the queue's
 * internal pointers. This routine returns the number of entries that were
 * consumed by the HBA.
 **/
static uint32_t
lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
{
        if ((hq->type != LPFC_HRQ) || (dq->type != LPFC_DRQ))
                return 0;
        hq->hba_index = ((hq->hba_index + 1) % hq->entry_count);
        dq->hba_index = ((dq->hba_index + 1) % dq->entry_count);
        return 1;
}

/**
 * lpfc_cmd_iocb - Get next command iocb entry in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function returns pointer to next command iocb entry
 * in the command ring. The caller must hold hbalock to prevent
 * other threads consume the next command iocb.
 * SLI-2/SLI-3 provide different sized iocbs.
 **/
static inline IOCB_t *
lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        return (IOCB_t *) (((char *) pring->cmdringaddr) +
                           pring->cmdidx * phba->iocb_cmd_size);
}

/**
 * lpfc_resp_iocb - Get next response iocb entry in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function returns pointer to next response iocb entry
 * in the response ring. The caller must hold hbalock to make sure
 * that no other thread consume the next response iocb.
 * SLI-2/SLI-3 provide different sized iocbs.
 **/
static inline IOCB_t *
lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        return (IOCB_t *) (((char *) pring->rspringaddr) +
                           pring->rspidx * phba->iocb_rsp_size);
}

/**
 * __lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
 * @phba: Pointer to HBA context object.
 *
 * This function is called with hbalock held. This function
 * allocates a new driver iocb object from the iocb pool. If the
 * allocation is successful, it returns pointer to the newly
 * allocated iocb object else it returns NULL.
 **/
static struct lpfc_iocbq *
__lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{
        struct list_head *lpfc_iocb_list = &phba->lpfc_iocb_list;
        struct lpfc_iocbq * iocbq = NULL;

        list_remove_head(lpfc_iocb_list, iocbq, struct lpfc_iocbq, list);
        return iocbq;
}

/**
 * __lpfc_clear_active_sglq - Remove the active sglq for this XRI.
 * @phba: Pointer to HBA context object.
 * @xritag: XRI value.
 *
 * This function clears the sglq pointer from the array of acive
 * sglq's. The xritag that is passed in is used to index into the
 * array. Before the xritag can be used it needs to be adjusted
 * by subtracting the xribase.
 *
 * Returns sglq ponter = success, NULL = Failure.
 **/
static struct lpfc_sglq *
__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{
        uint16_t adj_xri;
        struct lpfc_sglq *sglq;
        adj_xri = xritag - phba->sli4_hba.max_cfg_param.xri_base;
        if (adj_xri > phba->sli4_hba.max_cfg_param.max_xri)
                return NULL;
        sglq = phba->sli4_hba.lpfc_sglq_active_list[adj_xri];
        phba->sli4_hba.lpfc_sglq_active_list[adj_xri] = NULL;
        return sglq;
}

/**
 * __lpfc_get_active_sglq - Get the active sglq for this XRI.
 * @phba: Pointer to HBA context object.
 * @xritag: XRI value.
 *
 * This function returns the sglq pointer from the array of acive
 * sglq's. The xritag that is passed in is used to index into the
 * array. Before the xritag can be used it needs to be adjusted
 * by subtracting the xribase.
 *
 * Returns sglq ponter = success, NULL = Failure.
 **/
static struct lpfc_sglq *
__lpfc_get_active_sglq(struct lpfc_hba *phba, uint16_t xritag)
{
        uint16_t adj_xri;
        struct lpfc_sglq *sglq;
        adj_xri = xritag - phba->sli4_hba.max_cfg_param.xri_base;
        if (adj_xri > phba->sli4_hba.max_cfg_param.max_xri)
                return NULL;
        sglq =  phba->sli4_hba.lpfc_sglq_active_list[adj_xri];
        return sglq;
}

/**
 * __lpfc_sli_get_sglq - Allocates an iocb object from sgl pool
 * @phba: Pointer to HBA context object.
 *
 * This function is called with hbalock held. This function
 * Gets a new driver sglq object from the sglq list. If the
 * list is not empty then it is successful, it returns pointer to the newly
 * allocated sglq object else it returns NULL.
 **/
static struct lpfc_sglq *
__lpfc_sli_get_sglq(struct lpfc_hba *phba)
{
        struct list_head *lpfc_sgl_list = &phba->sli4_hba.lpfc_sgl_list;
        struct lpfc_sglq *sglq = NULL;
        uint16_t adj_xri;
        list_remove_head(lpfc_sgl_list, sglq, struct lpfc_sglq, list);
        adj_xri = sglq->sli4_xritag - phba->sli4_hba.max_cfg_param.xri_base;
        phba->sli4_hba.lpfc_sglq_active_list[adj_xri] = sglq;
        return sglq;
}

/**
 * lpfc_sli_get_iocbq - Allocates an iocb object from iocb pool
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held. This function
 * allocates a new driver iocb object from the iocb pool. If the
 * allocation is successful, it returns pointer to the newly
 * allocated iocb object else it returns NULL.
 **/
struct lpfc_iocbq *
lpfc_sli_get_iocbq(struct lpfc_hba *phba)
{
        struct lpfc_iocbq * iocbq = NULL;
        unsigned long iflags;

        spin_lock_irqsave(&phba->hbalock, iflags);
        iocbq = __lpfc_sli_get_iocbq(phba);
        spin_unlock_irqrestore(&phba->hbalock, iflags);
        return iocbq;
}

/**
 * __lpfc_sli_release_iocbq_s4 - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 * The sqlq structure that holds the xritag and phys and virtual
 * mappings for the scatter gather list is retrieved from the
 * active array of sglq. The get of the sglq pointer also clears
 * the entry in the array. If the status of the IO indiactes that
 * this IO was aborted then the sglq entry it put on the
 * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
 * IO has good status or fails for any other reason then the sglq
 * entry is added to the free list (lpfc_sgl_list).
 **/
static void
__lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
        struct lpfc_sglq *sglq;
        size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
        unsigned long iflag;

        if (iocbq->sli4_xritag == NO_XRI)
                sglq = NULL;
        else
                sglq = __lpfc_clear_active_sglq(phba, iocbq->sli4_xritag);
        if (sglq)  {
                if (iocbq->iocb_flag & LPFC_DRIVER_ABORTED
                        || ((iocbq->iocb.ulpStatus == IOSTAT_LOCAL_REJECT)
                        && (iocbq->iocb.un.ulpWord[4]
                                == IOERR_SLI_ABORTED))) {
                        spin_lock_irqsave(&phba->sli4_hba.abts_sgl_list_lock,
                                        iflag);
                        list_add(&sglq->list,
                                &phba->sli4_hba.lpfc_abts_els_sgl_list);
                        spin_unlock_irqrestore(
                                &phba->sli4_hba.abts_sgl_list_lock, iflag);
                } else
                        list_add(&sglq->list, &phba->sli4_hba.lpfc_sgl_list);
        }


        /*
         * Clean all volatile data fields, preserve iotag and node struct.
         */
        memset((char *)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
        iocbq->sli4_xritag = NO_XRI;
        list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}

/**
 * __lpfc_sli_release_iocbq_s3 - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 **/
static void
__lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
        size_t start_clean = offsetof(struct lpfc_iocbq, iocb);

        /*
         * Clean all volatile data fields, preserve iotag and node struct.
         */
        memset((char*)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean);
        iocbq->sli4_xritag = NO_XRI;
        list_add_tail(&iocbq->list, &phba->lpfc_iocb_list);
}

/**
 * __lpfc_sli_release_iocbq - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with hbalock held to release driver
 * iocb object to the iocb pool. The iotag in the iocb object
 * does not change for each use of the iocb object. This function
 * clears all other fields of the iocb object when it is freed.
 **/
static void
__lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
        phba->__lpfc_sli_release_iocbq(phba, iocbq);
}

/**
 * lpfc_sli_release_iocbq - Release iocb to the iocb pool
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function is called with no lock held to release the iocb to
 * iocb pool.
 **/
void
lpfc_sli_release_iocbq(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
        unsigned long iflags;

        /*
         * Clean all volatile data fields, preserve iotag and node struct.
         */
        spin_lock_irqsave(&phba->hbalock, iflags);
        __lpfc_sli_release_iocbq(phba, iocbq);
        spin_unlock_irqrestore(&phba->hbalock, iflags);
}

/**
 * lpfc_sli_cancel_iocbs - Cancel all iocbs from a list.
 * @phba: Pointer to HBA context object.
 * @iocblist: List of IOCBs.
 * @ulpstatus: ULP status in IOCB command field.
 * @ulpWord4: ULP word-4 in IOCB command field.
 *
 * This function is called with a list of IOCBs to cancel. It cancels the IOCB
 * on the list by invoking the complete callback function associated with the
 * IOCB with the provided @ulpstatus and @ulpword4 set to the IOCB commond
 * fields.
 **/
void
lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist,
                      uint32_t ulpstatus, uint32_t ulpWord4)
{
        struct lpfc_iocbq *piocb;

        while (!list_empty(iocblist)) {
                list_remove_head(iocblist, piocb, struct lpfc_iocbq, list);

                if (!piocb->iocb_cmpl)
                        lpfc_sli_release_iocbq(phba, piocb);
                else {
                        piocb->iocb.ulpStatus = ulpstatus;
                        piocb->iocb.un.ulpWord[4] = ulpWord4;
                        (piocb->iocb_cmpl) (phba, piocb, piocb);
                }
        }
        return;
}

/**
 * lpfc_sli_iocb_cmd_type - Get the iocb type
 * @iocb_cmnd: iocb command code.
 *
 * This function is called by ring event handler function to get the iocb type.
 * This function translates the iocb command to an iocb command type used to
 * decide the final disposition of each completed IOCB.
 * The function returns
 * LPFC_UNKNOWN_IOCB if it is an unsupported iocb
 * LPFC_SOL_IOCB     if it is a solicited iocb completion
 * LPFC_ABORT_IOCB   if it is an abort iocb
 * LPFC_UNSOL_IOCB   if it is an unsolicited iocb
 *
 * The caller is not required to hold any lock.
 **/
static lpfc_iocb_type
lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd)
{
        lpfc_iocb_type type = LPFC_UNKNOWN_IOCB;

        if (iocb_cmnd > CMD_MAX_IOCB_CMD)
                return 0;

        switch (iocb_cmnd) {
        case CMD_XMIT_SEQUENCE_CR:
        case CMD_XMIT_SEQUENCE_CX:
        case CMD_XMIT_BCAST_CN:
        case CMD_XMIT_BCAST_CX:
        case CMD_ELS_REQUEST_CR:
        case CMD_ELS_REQUEST_CX:
        case CMD_CREATE_XRI_CR:
        case CMD_CREATE_XRI_CX:
        case CMD_GET_RPI_CN:
        case CMD_XMIT_ELS_RSP_CX:
        case CMD_GET_RPI_CR:
        case CMD_FCP_IWRITE_CR:
        case CMD_FCP_IWRITE_CX:
        case CMD_FCP_IREAD_CR:
        case CMD_FCP_IREAD_CX:
        case CMD_FCP_ICMND_CR:
        case CMD_FCP_ICMND_CX:
        case CMD_FCP_TSEND_CX:
        case CMD_FCP_TRSP_CX:
        case CMD_FCP_TRECEIVE_CX:
        case CMD_FCP_AUTO_TRSP_CX:
        case CMD_ADAPTER_MSG:
        case CMD_ADAPTER_DUMP:
        case CMD_XMIT_SEQUENCE64_CR:
        case CMD_XMIT_SEQUENCE64_CX:
        case CMD_XMIT_BCAST64_CN:
        case CMD_XMIT_BCAST64_CX:
        case CMD_ELS_REQUEST64_CR:
        case CMD_ELS_REQUEST64_CX:
        case CMD_FCP_IWRITE64_CR:
        case CMD_FCP_IWRITE64_CX:
        case CMD_FCP_IREAD64_CR:
        case CMD_FCP_IREAD64_CX:
        case CMD_FCP_ICMND64_CR:
        case CMD_FCP_ICMND64_CX:
        case CMD_FCP_TSEND64_CX:
        case CMD_FCP_TRSP64_CX:
        case CMD_FCP_TRECEIVE64_CX:
        case CMD_GEN_REQUEST64_CR:
        case CMD_GEN_REQUEST64_CX:
        case CMD_XMIT_ELS_RSP64_CX:
        case DSSCMD_IWRITE64_CR:
        case DSSCMD_IWRITE64_CX:
        case DSSCMD_IREAD64_CR:
        case DSSCMD_IREAD64_CX:
        case DSSCMD_INVALIDATE_DEK:
        case DSSCMD_SET_KEK:
        case DSSCMD_GET_KEK_ID:
        case DSSCMD_GEN_XFER:
                type = LPFC_SOL_IOCB;
                break;
        case CMD_ABORT_XRI_CN:
        case CMD_ABORT_XRI_CX:
        case CMD_CLOSE_XRI_CN:
        case CMD_CLOSE_XRI_CX:
        case CMD_XRI_ABORTED_CX:
        case CMD_ABORT_MXRI64_CN:
                type = LPFC_ABORT_IOCB;
                break;
        case CMD_RCV_SEQUENCE_CX:
        case CMD_RCV_ELS_REQ_CX:
        case CMD_RCV_SEQUENCE64_CX:
        case CMD_RCV_ELS_REQ64_CX:
        case CMD_ASYNC_STATUS:
        case CMD_IOCB_RCV_SEQ64_CX:
        case CMD_IOCB_RCV_ELS64_CX:
        case CMD_IOCB_RCV_CONT64_CX:
        case CMD_IOCB_RET_XRI64_CX:
                type = LPFC_UNSOL_IOCB;
                break;
        case CMD_IOCB_XMIT_MSEQ64_CR:
        case CMD_IOCB_XMIT_MSEQ64_CX:
        case CMD_IOCB_RCV_SEQ_LIST64_CX:
        case CMD_IOCB_RCV_ELS_LIST64_CX:
        case CMD_IOCB_CLOSE_EXTENDED_CN:
        case CMD_IOCB_ABORT_EXTENDED_CN:
        case CMD_IOCB_RET_HBQE64_CN:
        case CMD_IOCB_FCP_IBIDIR64_CR:
        case CMD_IOCB_FCP_IBIDIR64_CX:
        case CMD_IOCB_FCP_ITASKMGT64_CX:
        case CMD_IOCB_LOGENTRY_CN:
        case CMD_IOCB_LOGENTRY_ASYNC_CN:
                printk("%s - Unhandled SLI-3 Command x%x\n",
                                __func__, iocb_cmnd);
                type = LPFC_UNKNOWN_IOCB;
                break;
        default:
                type = LPFC_UNKNOWN_IOCB;
                break;
        }

        return type;
}

/**
 * lpfc_sli_ring_map - Issue config_ring mbox for all rings
 * @phba: Pointer to HBA context object.
 *
 * This function is called from SLI initialization code
 * to configure every ring of the HBA's SLI interface. The
 * caller is not required to hold any lock. This function issues
 * a config_ring mailbox command for each ring.
 * This function returns zero if successful else returns a negative
 * error code.
 **/
static int
lpfc_sli_ring_map(struct lpfc_hba *phba)
{
        struct lpfc_sli *psli = &phba->sli;
        LPFC_MBOXQ_t *pmb;
        MAILBOX_t *pmbox;
        int i, rc, ret = 0;

        pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
        if (!pmb)
                return -ENOMEM;
        pmbox = &pmb->u.mb;
        phba->link_state = LPFC_INIT_MBX_CMDS;
        for (i = 0; i < psli->num_rings; i++) {
                lpfc_config_ring(phba, i, pmb);
                rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
                if (rc != MBX_SUCCESS) {
                        lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
                                        "0446 Adapter failed to init (%d), "
                                        "mbxCmd x%x CFG_RING, mbxStatus x%x, "
                                        "ring %d\n",
                                        rc, pmbox->mbxCommand,
                                        pmbox->mbxStatus, i);
                        phba->link_state = LPFC_HBA_ERROR;
                        ret = -ENXIO;
                        break;
                }
        }
        mempool_free(pmb, phba->mbox_mem_pool);
        return ret;
}

/**
 * lpfc_sli_ringtxcmpl_put - Adds new iocb to the txcmplq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @piocb: Pointer to the driver iocb object.
 *
 * This function is called with hbalock held. The function adds the
 * new iocb to txcmplq of the given ring. This function always returns
 * 0. If this function is called for ELS ring, this function checks if
 * there is a vport associated with the ELS command. This function also
 * starts els_tmofunc timer if this is an ELS command.
 **/
static int
lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                        struct lpfc_iocbq *piocb)
{
        list_add_tail(&piocb->list, &pring->txcmplq);
        pring->txcmplq_cnt++;
        if ((unlikely(pring->ringno == LPFC_ELS_RING)) &&
           (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) &&
           (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) {
                if (!piocb->vport)
                        BUG();
                else
                        mod_timer(&piocb->vport->els_tmofunc,
                                  jiffies + HZ * (phba->fc_ratov << 1));
        }


        return 0;
}

/**
 * lpfc_sli_ringtx_get - Get first element of the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held to get next
 * iocb in txq of the given ring. If there is any iocb in
 * the txq, the function returns first iocb in the list after
 * removing the iocb from the list, else it returns NULL.
 **/
static struct lpfc_iocbq *
lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        struct lpfc_iocbq *cmd_iocb;

        list_remove_head((&pring->txq), cmd_iocb, struct lpfc_iocbq, list);
        if (cmd_iocb != NULL)
                pring->txq_cnt--;
        return cmd_iocb;
}

/**
 * lpfc_sli_next_iocb_slot - Get next iocb slot in the ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held and the caller must post the
 * iocb without releasing the lock. If the caller releases the lock,
 * iocb slot returned by the function is not guaranteed to be available.
 * The function returns pointer to the next available iocb slot if there
 * is available slot in the ring, else it returns NULL.
 * If the get index of the ring is ahead of the put index, the function
 * will post an error attention event to the worker thread to take the
 * HBA to offline state.
 **/
static IOCB_t *
lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
        uint32_t  max_cmd_idx = pring->numCiocb;
        if ((pring->next_cmdidx == pring->cmdidx) &&
           (++pring->next_cmdidx >= max_cmd_idx))
                pring->next_cmdidx = 0;

        if (unlikely(pring->local_getidx == pring->next_cmdidx)) {

                pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);

                if (unlikely(pring->local_getidx >= max_cmd_idx)) {
                        lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                                        "0315 Ring %d issue: portCmdGet %d "
                                        "is bigger than cmd ring %d\n",
                                        pring->ringno,
                                        pring->local_getidx, max_cmd_idx);

                        phba->link_state = LPFC_HBA_ERROR;
                        /*
                         * All error attention handlers are posted to
                         * worker thread
                         */
                        phba->work_ha |= HA_ERATT;
                        phba->work_hs = HS_FFER3;

                        lpfc_worker_wake_up(phba);

                        return NULL;
                }

                if (pring->local_getidx == pring->next_cmdidx)
                        return NULL;
        }

        return lpfc_cmd_iocb(phba, pring);
}

/**
 * lpfc_sli_next_iotag - Get an iotag for the iocb
 * @phba: Pointer to HBA context object.
 * @iocbq: Pointer to driver iocb object.
 *
 * This function gets an iotag for the iocb. If there is no unused iotag and
 * the iocbq_lookup_len < 0xffff, this function allocates a bigger iotag_lookup
 * array and assigns a new iotag.
 * The function returns the allocated iotag if successful, else returns zero.
 * Zero is not a valid iotag.
 * The caller is not required to hold any lock.
 **/
uint16_t
lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
{
        struct lpfc_iocbq **new_arr;
        struct lpfc_iocbq **old_arr;
        size_t new_len;
        struct lpfc_sli *psli = &phba->sli;
        uint16_t iotag;

        spin_lock_irq(&phba->hbalock);
        iotag = psli->last_iotag;
        if(++iotag < psli->iocbq_lookup_len) {
                psli->last_iotag = iotag;
                psli->iocbq_lookup[iotag] = iocbq;
                spin_unlock_irq(&phba->hbalock);
                iocbq->iotag = iotag;
                return iotag;
        } else if (psli->iocbq_lookup_len < (0xffff
                                           - LPFC_IOCBQ_LOOKUP_INCREMENT)) {
                new_len = psli->iocbq_lookup_len + LPFC_IOCBQ_LOOKUP_INCREMENT;
                spin_unlock_irq(&phba->hbalock);
                new_arr = kzalloc(new_len * sizeof (struct lpfc_iocbq *),
                                  GFP_KERNEL);
                if (new_arr) {
                        spin_lock_irq(&phba->hbalock);
                        old_arr = psli->iocbq_lookup;
                        if (new_len <= psli->iocbq_lookup_len) {
                                /* highly unprobable case */
                                kfree(new_arr);
                                iotag = psli->last_iotag;
                                if(++iotag < psli->iocbq_lookup_len) {
                                        psli->last_iotag = iotag;

                                        psli->iocbq_lookup[iotag] = iocbq;
                                        spin_unlock_irq(&phba->hbalock);
                                        iocbq->iotag = iotag;
                                        return iotag;
                                }
                                spin_unlock_irq(&phba->hbalock);
                                return 0;
                        }
                        if (psli->iocbq_lookup)
                                memcpy(new_arr, old_arr,
                                       ((psli->last_iotag  + 1) *
                                        sizeof (struct lpfc_iocbq *)));
                        psli->iocbq_lookup = new_arr;
                        psli->iocbq_lookup_len = new_len;
                        psli->last_iotag = iotag;
                        psli->iocbq_lookup[iotag] = iocbq;
                        spin_unlock_irq(&phba->hbalock);
                        iocbq->iotag = iotag;
                        kfree(old_arr);
                        return iotag;
                }
        } else
                spin_unlock_irq(&phba->hbalock);

        lpfc_printf_log(phba, KERN_ERR,LOG_SLI,
                        "0318 Failed to allocate IOTAG.last IOTAG is %d\n",
                        psli->last_iotag);

        return 0;
}

/**
 * lpfc_sli_submit_iocb - Submit an iocb to the firmware
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @iocb: Pointer to iocb slot in the ring.
 * @nextiocb: Pointer to driver iocb object which need to be
 *            posted to firmware.
 *
 * This function is called with hbalock held to post a new iocb to
 * the firmware. This function copies the new iocb to ring iocb slot and
 * updates the ring pointers. It adds the new iocb to txcmplq if there is
 * a completion call back for this iocb else the function will free the
 * iocb object.
 **/
static void
lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                IOCB_t *iocb, struct lpfc_iocbq *nextiocb)
{
        /*
         * Set up an iotag
         */
        nextiocb->iocb.ulpIoTag = (nextiocb->iocb_cmpl) ? nextiocb->iotag : 0;


        if (pring->ringno == LPFC_ELS_RING) {
                lpfc_debugfs_slow_ring_trc(phba,
                        "IOCB cmd ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
                        *(((uint32_t *) &nextiocb->iocb) + 4),
                        *(((uint32_t *) &nextiocb->iocb) + 6),
                        *(((uint32_t *) &nextiocb->iocb) + 7));
        }

        /*
         * Issue iocb command to adapter
         */
        lpfc_sli_pcimem_bcopy(&nextiocb->iocb, iocb, phba->iocb_cmd_size);
        wmb();
        pring->stats.iocb_cmd++;

        /*
         * If there is no completion routine to call, we can release the
         * IOCB buffer back right now. For IOCBs, like QUE_RING_BUF,
         * that have no rsp ring completion, iocb_cmpl MUST be NULL.
         */
        if (nextiocb->iocb_cmpl)
                lpfc_sli_ringtxcmpl_put(phba, pring, nextiocb);
        else
                __lpfc_sli_release_iocbq(phba, nextiocb);

        /*
         * Let the HBA know what IOCB slot will be the next one the
         * driver will put a command into.
         */
        pring->cmdidx = pring->next_cmdidx;
        writel(pring->cmdidx, &phba->host_gp[pring->ringno].cmdPutInx);
}

/**
 * lpfc_sli_update_full_ring - Update the chip attention register
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * The caller is not required to hold any lock for calling this function.
 * This function updates the chip attention bits for the ring to inform firmware
 * that there are pending work to be done for this ring and requests an
 * interrupt when there is space available in the ring. This function is
 * called when the driver is unable to post more iocbs to the ring due
 * to unavailability of space in the ring.
 **/
static void
lpfc_sli_update_full_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        int ringno = pring->ringno;

        pring->flag |= LPFC_CALL_RING_AVAILABLE;

        wmb();

        /*
         * Set ring 'ringno' to SET R0CE_REQ in Chip Att register.
         * The HBA will tell us when an IOCB entry is available.
         */
        writel((CA_R0ATT|CA_R0CE_REQ) << (ringno*4), phba->CAregaddr);
        readl(phba->CAregaddr); /* flush */

        pring->stats.iocb_cmd_full++;
}

/**
 * lpfc_sli_update_ring - Update chip attention register
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function updates the chip attention register bit for the
 * given ring to inform HBA that there is more work to be done
 * in this ring. The caller is not required to hold any lock.
 **/
static void
lpfc_sli_update_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        int ringno = pring->ringno;

        /*
         * Tell the HBA that there is work to do in this ring.
         */
        if (!(phba->sli3_options & LPFC_SLI3_CRP_ENABLED)) {
                wmb();
                writel(CA_R0ATT << (ringno * 4), phba->CAregaddr);
                readl(phba->CAregaddr); /* flush */
        }
}

/**
 * lpfc_sli_resume_iocb - Process iocbs in the txq
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called with hbalock held to post pending iocbs
 * in the txq to the firmware. This function is called when driver
 * detects space available in the ring.
 **/
static void
lpfc_sli_resume_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        IOCB_t *iocb;
        struct lpfc_iocbq *nextiocb;

        /*
         * Check to see if:
         *  (a) there is anything on the txq to send
         *  (b) link is up
         *  (c) link attention events can be processed (fcp ring only)
         *  (d) IOCB processing is not blocked by the outstanding mbox command.
         */
        if (pring->txq_cnt &&
            lpfc_is_link_up(phba) &&
            (pring->ringno != phba->sli.fcp_ring ||
             phba->sli.sli_flag & LPFC_PROCESS_LA)) {

                while ((iocb = lpfc_sli_next_iocb_slot(phba, pring)) &&
                       (nextiocb = lpfc_sli_ringtx_get(phba, pring)))
                        lpfc_sli_submit_iocb(phba, pring, iocb, nextiocb);

                if (iocb)
                        lpfc_sli_update_ring(phba, pring);
                else
                        lpfc_sli_update_full_ring(phba, pring);
        }

        return;
}

/**
 * lpfc_sli_next_hbq_slot - Get next hbq entry for the HBQ
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 *
 * This function is called with hbalock held to get the next
 * available slot for the given HBQ. If there is free slot
 * available for the HBQ it will return pointer to the next available
 * HBQ entry else it will return NULL.
 **/
static struct lpfc_hbq_entry *
lpfc_sli_next_hbq_slot(struct lpfc_hba *phba, uint32_t hbqno)
{
        struct hbq_s *hbqp = &phba->hbqs[hbqno];

        if (hbqp->next_hbqPutIdx == hbqp->hbqPutIdx &&
            ++hbqp->next_hbqPutIdx >= hbqp->entry_count)
                hbqp->next_hbqPutIdx = 0;

        if (unlikely(hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)) {
                uint32_t raw_index = phba->hbq_get[hbqno];
                uint32_t getidx = le32_to_cpu(raw_index);

                hbqp->local_hbqGetIdx = getidx;

                if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) {
                        lpfc_printf_log(phba, KERN_ERR,
                                        LOG_SLI | LOG_VPORT,
                                        "1802 HBQ %d: local_hbqGetIdx "
                                        "%u is > than hbqp->entry_count %u\n",
                                        hbqno, hbqp->local_hbqGetIdx,
                                        hbqp->entry_count);

                        phba->link_state = LPFC_HBA_ERROR;
                        return NULL;
                }

                if (hbqp->local_hbqGetIdx == hbqp->next_hbqPutIdx)
                        return NULL;
        }

        return (struct lpfc_hbq_entry *) phba->hbqs[hbqno].hbq_virt +
                        hbqp->hbqPutIdx;
}

/**
 * lpfc_sli_hbqbuf_free_all - Free all the hbq buffers
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held to free all the
 * hbq buffers while uninitializing the SLI interface. It also
 * frees the HBQ buffers returned by the firmware but not yet
 * processed by the upper layers.
 **/
void
lpfc_sli_hbqbuf_free_all(struct lpfc_hba *phba)
{
        struct lpfc_dmabuf *dmabuf, *next_dmabuf;
        struct hbq_dmabuf *hbq_buf;
        unsigned long flags;
        int i, hbq_count;
        uint32_t hbqno;

        hbq_count = lpfc_sli_hbq_count();
        /* Return all memory used by all HBQs */
        spin_lock_irqsave(&phba->hbalock, flags);
        for (i = 0; i < hbq_count; ++i) {
                list_for_each_entry_safe(dmabuf, next_dmabuf,
                                &phba->hbqs[i].hbq_buffer_list, list) {
                        hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
                        list_del(&hbq_buf->dbuf.list);
                        (phba->hbqs[i].hbq_free_buffer)(phba, hbq_buf);
                }
                phba->hbqs[i].buffer_count = 0;
        }
        /* Return all HBQ buffer that are in-fly */
        list_for_each_entry_safe(dmabuf, next_dmabuf, &phba->rb_pend_list,
                                 list) {
                hbq_buf = container_of(dmabuf, struct hbq_dmabuf, dbuf);
                list_del(&hbq_buf->dbuf.list);
                if (hbq_buf->tag == -1) {
                        (phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer)
                                (phba, hbq_buf);
                } else {
                        hbqno = hbq_buf->tag >> 16;
                        if (hbqno >= LPFC_MAX_HBQS)
                                (phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer)
                                        (phba, hbq_buf);
                        else
                                (phba->hbqs[hbqno].hbq_free_buffer)(phba,
                                        hbq_buf);
                }
        }

        /* Mark the HBQs not in use */
        phba->hbq_in_use = 0;
        spin_unlock_irqrestore(&phba->hbalock, flags);
}

/**
 * lpfc_sli_hbq_to_firmware - Post the hbq buffer to firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post a
 * hbq buffer to the firmware. If the function finds an empty
 * slot in the HBQ, it will post the buffer. The function will return
 * pointer to the hbq entry if it successfully post the buffer
 * else it will return NULL.
 **/
static int
lpfc_sli_hbq_to_firmware(struct lpfc_hba *phba, uint32_t hbqno,
                         struct hbq_dmabuf *hbq_buf)
{
        return phba->lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buf);
}

/**
 * lpfc_sli_hbq_to_firmware_s3 - Post the hbq buffer to SLI3 firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post a hbq buffer to the
 * firmware. If the function finds an empty slot in the HBQ, it will post the
 * buffer and place it on the hbq_buffer_list. The function will return zero if
 * it successfully post the buffer else it will return an error.
 **/
static int
lpfc_sli_hbq_to_firmware_s3(struct lpfc_hba *phba, uint32_t hbqno,
                            struct hbq_dmabuf *hbq_buf)
{
        struct lpfc_hbq_entry *hbqe;
        dma_addr_t physaddr = hbq_buf->dbuf.phys;

        /* Get next HBQ entry slot to use */
        hbqe = lpfc_sli_next_hbq_slot(phba, hbqno);
        if (hbqe) {
                struct hbq_s *hbqp = &phba->hbqs[hbqno];

                hbqe->bde.addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
                hbqe->bde.addrLow  = le32_to_cpu(putPaddrLow(physaddr));
                hbqe->bde.tus.f.bdeSize = hbq_buf->size;
                hbqe->bde.tus.f.bdeFlags = 0;
                hbqe->bde.tus.w = le32_to_cpu(hbqe->bde.tus.w);
                hbqe->buffer_tag = le32_to_cpu(hbq_buf->tag);
                                /* Sync SLIM */
                hbqp->hbqPutIdx = hbqp->next_hbqPutIdx;
                writel(hbqp->hbqPutIdx, phba->hbq_put + hbqno);
                                /* flush */
                readl(phba->hbq_put + hbqno);
                list_add_tail(&hbq_buf->dbuf.list, &hbqp->hbq_buffer_list);
                return 0;
        } else
                return -ENOMEM;
}

/**
 * lpfc_sli_hbq_to_firmware_s4 - Post the hbq buffer to SLI4 firmware
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @hbq_buf: Pointer to HBQ buffer.
 *
 * This function is called with the hbalock held to post an RQE to the SLI4
 * firmware. If able to post the RQE to the RQ it will queue the hbq entry to
 * the hbq_buffer_list and return zero, otherwise it will return an error.
 **/
static int
lpfc_sli_hbq_to_firmware_s4(struct lpfc_hba *phba, uint32_t hbqno,
                            struct hbq_dmabuf *hbq_buf)
{
        int rc;
        struct lpfc_rqe hrqe;
        struct lpfc_rqe drqe;

        hrqe.address_lo = putPaddrLow(hbq_buf->hbuf.phys);
        hrqe.address_hi = putPaddrHigh(hbq_buf->hbuf.phys);
        drqe.address_lo = putPaddrLow(hbq_buf->dbuf.phys);
        drqe.address_hi = putPaddrHigh(hbq_buf->dbuf.phys);
        rc = lpfc_sli4_rq_put(phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq,
                              &hrqe, &drqe);
        if (rc < 0)
                return rc;
        hbq_buf->tag = rc;
        list_add_tail(&hbq_buf->dbuf.list, &phba->hbqs[hbqno].hbq_buffer_list);
        return 0;
}

/* HBQ for ELS and CT traffic. */
static struct lpfc_hbq_init lpfc_els_hbq = {
        .rn = 1,
        .entry_count = 200,
        .mask_count = 0,
        .profile = 0,
        .ring_mask = (1 << LPFC_ELS_RING),
        .buffer_count = 0,
        .init_count = 40,
        .add_count = 40,
};

/* HBQ for the extra ring if needed */
static struct lpfc_hbq_init lpfc_extra_hbq = {
        .rn = 1,
        .entry_count = 200,
        .mask_count = 0,
        .profile = 0,
        .ring_mask = (1 << LPFC_EXTRA_RING),
        .buffer_count = 0,
        .init_count = 0,
        .add_count = 5,
};

/* Array of HBQs */
struct lpfc_hbq_init *lpfc_hbq_defs[] = {
        &lpfc_els_hbq,
        &lpfc_extra_hbq,
};

/**
 * lpfc_sli_hbqbuf_fill_hbqs - Post more hbq buffers to HBQ
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 * @count: Number of HBQ buffers to be posted.
 *
 * This function is called with no lock held to post more hbq buffers to the
 * given HBQ. The function returns the number of HBQ buffers successfully
 * posted.
 **/
static int
lpfc_sli_hbqbuf_fill_hbqs(struct lpfc_hba *phba, uint32_t hbqno, uint32_t count)
{
        uint32_t i, posted = 0;
        unsigned long flags;
        struct hbq_dmabuf *hbq_buffer;
        LIST_HEAD(hbq_buf_list);
        if (!phba->hbqs[hbqno].hbq_alloc_buffer)
                return 0;

        if ((phba->hbqs[hbqno].buffer_count + count) >
            lpfc_hbq_defs[hbqno]->entry_count)
                count = lpfc_hbq_defs[hbqno]->entry_count -
                                        phba->hbqs[hbqno].buffer_count;
        if (!count)
                return 0;
        /* Allocate HBQ entries */
        for (i = 0; i < count; i++) {
                hbq_buffer = (phba->hbqs[hbqno].hbq_alloc_buffer)(phba);
                if (!hbq_buffer)
                        break;
                list_add_tail(&hbq_buffer->dbuf.list, &hbq_buf_list);
        }
        /* Check whether HBQ is still in use */
        spin_lock_irqsave(&phba->hbalock, flags);
        if (!phba->hbq_in_use)
                goto err;
        while (!list_empty(&hbq_buf_list)) {
                list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
                                 dbuf.list);
                hbq_buffer->tag = (phba->hbqs[hbqno].buffer_count |
                                      (hbqno << 16));
                if (!lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer)) {
                        phba->hbqs[hbqno].buffer_count++;
                        posted++;
                } else
                        (phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
        }
        spin_unlock_irqrestore(&phba->hbalock, flags);
        return posted;
err:
        spin_unlock_irqrestore(&phba->hbalock, flags);
        while (!list_empty(&hbq_buf_list)) {
                list_remove_head(&hbq_buf_list, hbq_buffer, struct hbq_dmabuf,
                                 dbuf.list);
                (phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
        }
        return 0;
}

/**
 * lpfc_sli_hbqbuf_add_hbqs - Post more HBQ buffers to firmware
 * @phba: Pointer to HBA context object.
 * @qno: HBQ number.
 *
 * This function posts more buffers to the HBQ. This function
 * is called with no lock held. The function returns the number of HBQ entries
 * successfully allocated.
 **/
int
lpfc_sli_hbqbuf_add_hbqs(struct lpfc_hba *phba, uint32_t qno)
{
        return(lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
                                         lpfc_hbq_defs[qno]->add_count));
}

/**
 * lpfc_sli_hbqbuf_init_hbqs - Post initial buffers to the HBQ
 * @phba: Pointer to HBA context object.
 * @qno:  HBQ queue number.
 *
 * This function is called from SLI initialization code path with
 * no lock held to post initial HBQ buffers to firmware. The
 * function returns the number of HBQ entries successfully allocated.
 **/
static int
lpfc_sli_hbqbuf_init_hbqs(struct lpfc_hba *phba, uint32_t qno)
{
        return(lpfc_sli_hbqbuf_fill_hbqs(phba, qno,
                                         lpfc_hbq_defs[qno]->init_count));
}

/**
 * lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list
 * @phba: Pointer to HBA context object.
 * @hbqno: HBQ number.
 *
 * This function removes the first hbq buffer on an hbq list and returns a
 * pointer to that buffer. If it finds no buffers on the list it returns NULL.
 **/
static struct hbq_dmabuf *
lpfc_sli_hbqbuf_get(struct list_head *rb_list)
{
        struct lpfc_dmabuf *d_buf;

        list_remove_head(rb_list, d_buf, struct lpfc_dmabuf, list);
        if (!d_buf)
                return NULL;
        return container_of(d_buf, struct hbq_dmabuf, dbuf);
}

/**
 * lpfc_sli_hbqbuf_find - Find the hbq buffer associated with a tag
 * @phba: Pointer to HBA context object.
 * @tag: Tag of the hbq buffer.
 *
 * This function is called with hbalock held. This function searches
 * for the hbq buffer associated with the given tag in the hbq buffer
 * list. If it finds the hbq buffer, it returns the hbq_buffer other wise
 * it returns NULL.
 **/
static struct hbq_dmabuf *
lpfc_sli_hbqbuf_find(struct lpfc_hba *phba, uint32_t tag)
{
        struct lpfc_dmabuf *d_buf;
        struct hbq_dmabuf *hbq_buf;
        uint32_t hbqno;

        hbqno = tag >> 16;
        if (hbqno >= LPFC_MAX_HBQS)
                return NULL;

        spin_lock_irq(&phba->hbalock);
        list_for_each_entry(d_buf, &phba->hbqs[hbqno].hbq_buffer_list, list) {
                hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
                if (hbq_buf->tag == tag) {
                        spin_unlock_irq(&phba->hbalock);
                        return hbq_buf;
                }
        }
        spin_unlock_irq(&phba->hbalock);
        lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_VPORT,
                        "1803 Bad hbq tag. Data: x%x x%x\n",
                        tag, phba->hbqs[tag >> 16].buffer_count);
        return NULL;
}

/**
 * lpfc_sli_free_hbq - Give back the hbq buffer to firmware
 * @phba: Pointer to HBA context object.
 * @hbq_buffer: Pointer to HBQ buffer.
 *
 * This function is called with hbalock. This function gives back
 * the hbq buffer to firmware. If the HBQ does not have space to
 * post the buffer, it will free the buffer.
 **/
void
lpfc_sli_free_hbq(struct lpfc_hba *phba, struct hbq_dmabuf *hbq_buffer)
{
        uint32_t hbqno;

        if (hbq_buffer) {
                hbqno = hbq_buffer->tag >> 16;
                if (lpfc_sli_hbq_to_firmware(phba, hbqno, hbq_buffer))
                        (phba->hbqs[hbqno].hbq_free_buffer)(phba, hbq_buffer);
        }
}

/**
 * lpfc_sli_chk_mbx_command - Check if the mailbox is a legitimate mailbox
 * @mbxCommand: mailbox command code.
 *
 * This function is called by the mailbox event handler function to verify
 * that the completed mailbox command is a legitimate mailbox command. If the
 * completed mailbox is not known to the function, it will return MBX_SHUTDOWN
 * and the mailbox event handler will take the HBA offline.
 **/
static int
lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
{
        uint8_t ret;

        switch (mbxCommand) {
        case MBX_LOAD_SM:
        case MBX_READ_NV:
        case MBX_WRITE_NV:
        case MBX_WRITE_VPARMS:
        case MBX_RUN_BIU_DIAG:
        case MBX_INIT_LINK:
        case MBX_DOWN_LINK:
        case MBX_CONFIG_LINK:
        case MBX_CONFIG_RING:
        case MBX_RESET_RING:
        case MBX_READ_CONFIG:
        case MBX_READ_RCONFIG:
        case MBX_READ_SPARM:
        case MBX_READ_STATUS:
        case MBX_READ_RPI:
        case MBX_READ_XRI:
        case MBX_READ_REV:
        case MBX_READ_LNK_STAT:
        case MBX_REG_LOGIN:
        case MBX_UNREG_LOGIN:
        case MBX_READ_LA:
        case MBX_CLEAR_LA:
        case MBX_DUMP_MEMORY:
        case MBX_DUMP_CONTEXT:
        case MBX_RUN_DIAGS:
        case MBX_RESTART:
        case MBX_UPDATE_CFG:
        case MBX_DOWN_LOAD:
        case MBX_DEL_LD_ENTRY:
        case MBX_RUN_PROGRAM:
        case MBX_SET_MASK:
        case MBX_SET_VARIABLE:
        case MBX_UNREG_D_ID:
        case MBX_KILL_BOARD:
        case MBX_CONFIG_FARP:
        case MBX_BEACON:
        case MBX_LOAD_AREA:
        case MBX_RUN_BIU_DIAG64:
        case MBX_CONFIG_PORT:
        case MBX_READ_SPARM64:
        case MBX_READ_RPI64:
        case MBX_REG_LOGIN64:
        case MBX_READ_LA64:
        case MBX_WRITE_WWN:
        case MBX_SET_DEBUG:
        case MBX_LOAD_EXP_ROM:
        case MBX_ASYNCEVT_ENABLE:
        case MBX_REG_VPI:
        case MBX_UNREG_VPI:
        case MBX_HEARTBEAT:
        case MBX_PORT_CAPABILITIES:
        case MBX_PORT_IOV_CONTROL:
        case MBX_SLI4_CONFIG:
        case MBX_SLI4_REQ_FTRS:
        case MBX_REG_FCFI:
        case MBX_UNREG_FCFI:
        case MBX_REG_VFI:
        case MBX_UNREG_VFI:
        case MBX_INIT_VPI:
        case MBX_INIT_VFI:
        case MBX_RESUME_RPI:
                ret = mbxCommand;
                break;
        default:
                ret = MBX_SHUTDOWN;
                break;
        }
        return ret;
}

/**
 * lpfc_sli_wake_mbox_wait - lpfc_sli_issue_mbox_wait mbox completion handler
 * @phba: Pointer to HBA context object.
 * @pmboxq: Pointer to mailbox command.
 *
 * This is completion handler function for mailbox commands issued from
 * lpfc_sli_issue_mbox_wait function. This function is called by the
 * mailbox event handler function with no lock held. This function
 * will wake up thread waiting on the wait queue pointed by context1
 * of the mailbox.
 **/
void
lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
{
        wait_queue_head_t *pdone_q;
        unsigned long drvr_flag;

        /*
         * If pdone_q is empty, the driver thread gave up waiting and
         * continued running.
         */
        pmboxq->mbox_flag |= LPFC_MBX_WAKE;
        spin_lock_irqsave(&phba->hbalock, drvr_flag);
        pdone_q = (wait_queue_head_t *) pmboxq->context1;
        if (pdone_q)
                wake_up_interruptible(pdone_q);
        spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
        return;
}


/**
 * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler
 * @phba: Pointer to HBA context object.
 * @pmb: Pointer to mailbox object.
 *
 * This function is the default mailbox completion handler. It
 * frees the memory resources associated with the completed mailbox
 * command. If the completed command is a REG_LOGIN mailbox command,
 * this function will issue a UREG_LOGIN to re-claim the RPI.
 **/
void
lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
{
        struct lpfc_dmabuf *mp;
        uint16_t rpi, vpi;
        int rc;

        mp = (struct lpfc_dmabuf *) (pmb->context1);

        if (mp) {
                lpfc_mbuf_free(phba, mp->virt, mp->phys);
                kfree(mp);
        }

        if ((pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) &&
            (phba->sli_rev == LPFC_SLI_REV4))
                lpfc_sli4_free_rpi(phba, pmb->u.mb.un.varUnregLogin.rpi);

        /*
         * If a REG_LOGIN succeeded  after node is destroyed or node
         * is in re-discovery driver need to cleanup the RPI.
         */
        if (!(phba->pport->load_flag & FC_UNLOADING) &&
            pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 &&
            !pmb->u.mb.mbxStatus) {
                rpi = pmb->u.mb.un.varWords[0];
                vpi = pmb->u.mb.un.varRegLogin.vpi - phba->vpi_base;
                lpfc_unreg_login(phba, vpi, rpi, pmb);
                pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
                rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
                if (rc != MBX_NOT_FINISHED)
                        return;
        }

        if (bf_get(lpfc_mqe_command, &pmb->u.mqe) == MBX_SLI4_CONFIG)
                lpfc_sli4_mbox_cmd_free(phba, pmb);
        else
                mempool_free(pmb, phba->mbox_mem_pool);
}

/**
 * lpfc_sli_handle_mb_event - Handle mailbox completions from firmware
 * @phba: Pointer to HBA context object.
 *
 * This function is called with no lock held. This function processes all
 * the completed mailbox commands and gives it to upper layers. The interrupt
 * service routine processes mailbox completion interrupt and adds completed
 * mailbox commands to the mboxq_cmpl queue and signals the worker thread.
 * Worker thread call lpfc_sli_handle_mb_event, which will return the
 * completed mailbox commands in mboxq_cmpl queue to the upper layers. This
 * function returns the mailbox commands to the upper layer by calling the
 * completion handler function of each mailbox.
 **/
int
lpfc_sli_handle_mb_event(struct lpfc_hba *phba)
{
        MAILBOX_t *pmbox;
        LPFC_MBOXQ_t *pmb;
        int rc;
        LIST_HEAD(cmplq);

        phba->sli.slistat.mbox_event++;

        /* Get all completed mailboxe buffers into the cmplq */
        spin_lock_irq(&phba->hbalock);
        list_splice_init(&phba->sli.mboxq_cmpl, &cmplq);
        spin_unlock_irq(&phba->hbalock);

        /* Get a Mailbox buffer to setup mailbox commands for callback */
        do {
                list_remove_head(&cmplq, pmb, LPFC_MBOXQ_t, list);
                if (pmb == NULL)
                        break;

                pmbox = &pmb->u.mb;

                if (pmbox->mbxCommand != MBX_HEARTBEAT) {
                        if (pmb->vport) {
                                lpfc_debugfs_disc_trc(pmb->vport,
                                        LPFC_DISC_TRC_MBOX_VPORT,
                                        "MBOX cmpl vport: cmd:x%x mb:x%x x%x",
                                        (uint32_t)pmbox->mbxCommand,
                                        pmbox->un.varWords[0],
                                        pmbox->un.varWords[1]);
                        }
                        else {
                                lpfc_debugfs_disc_trc(phba->pport,
                                        LPFC_DISC_TRC_MBOX,
                                        "MBOX cmpl:       cmd:x%x mb:x%x x%x",
                                        (uint32_t)pmbox->mbxCommand,
                                        pmbox->un.varWords[0],
                                        pmbox->un.varWords[1]);
                        }
                }

                /*
                 * It is a fatal error if unknown mbox command completion.
                 */
                if (lpfc_sli_chk_mbx_command(pmbox->mbxCommand) ==
                    MBX_SHUTDOWN) {
                        /* Unknow mailbox command compl */
                        lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI,
                                        "(%d):0323 Unknown Mailbox command "
                                        "x%x (x%x) Cmpl\n",
                                        pmb->vport ? pmb->vport->vpi : 0,
                                        pmbox->mbxCommand,
                                        lpfc_sli4_mbox_opcode_get(phba, pmb));
                        phba->link_state = LPFC_HBA_ERROR;
                        phba->work_hs = HS_FFER3;
                        lpfc_handle_eratt(phba);
                        continue;
                }

                if (pmbox->mbxStatus) {
                        phba->sli.slistat.mbox_stat_err++;
                        if (pmbox->mbxStatus == MBXERR_NO_RESOURCES) {
                                /* Mbox cmd cmpl error - RETRYing */
                                lpfc_printf_log(phba, KERN_INFO,
                                                LOG_MBOX | LOG_SLI,
                                                "(%d):0305 Mbox cmd cmpl "
                                                "error - RETRYing Data: x%x "
                                                "(x%x) x%x x%x x%x\n",
                                                pmb->vport ? pmb->vport->vpi :0,
                                                pmbox->mbxCommand,
                                                lpfc_sli4_mbox_opcode_get(phba,
                                                                          pmb),
                                                pmbox->mbxStatus,
                                                pmbox->un.varWords[0],
                                                pmb->vport->port_state);
                                pmbox->mbxStatus = 0;
                                pmbox->mbxOwner = OWN_HOST;
                                rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT);
                                if (rc != MBX_NOT_FINISHED)
                                        continue;
                        }
                }

                /* Mailbox cmd <cmd> Cmpl <cmpl> */
                lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
                                "(%d):0307 Mailbox cmd x%x (x%x) Cmpl x%p "
                                "Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x\n",
                                pmb->vport ? pmb->vport->vpi : 0,
                                pmbox->mbxCommand,
                                lpfc_sli4_mbox_opcode_get(phba, pmb),
                                pmb->mbox_cmpl,
                                *((uint32_t *) pmbox),
                                pmbox->un.varWords[0],
                                pmbox->un.varWords[1],
                                pmbox->un.varWords[2],
                                pmbox->un.varWords[3],
                                pmbox->un.varWords[4],
                                pmbox->un.varWords[5],
                                pmbox->un.varWords[6],
                                pmbox->un.varWords[7]);

                if (pmb->mbox_cmpl)
                        pmb->mbox_cmpl(phba,pmb);
        } while (1);
        return 0;
}

/**
 * lpfc_sli_get_buff - Get the buffer associated with the buffer tag
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @tag: buffer tag.
 *
 * This function is called with no lock held. When QUE_BUFTAG_BIT bit
 * is set in the tag the buffer is posted for a particular exchange,
 * the function will return the buffer without replacing the buffer.
 * If the buffer is for unsolicited ELS or CT traffic, this function
 * returns the buffer and also posts another buffer to the firmware.
 **/
static struct lpfc_dmabuf *
lpfc_sli_get_buff(struct lpfc_hba *phba,
                  struct lpfc_sli_ring *pring,
                  uint32_t tag)
{
        struct hbq_dmabuf *hbq_entry;

        if (tag & QUE_BUFTAG_BIT)
                return lpfc_sli_ring_taggedbuf_get(phba, pring, tag);
        hbq_entry = lpfc_sli_hbqbuf_find(phba, tag);
        if (!hbq_entry)
                return NULL;
        return &hbq_entry->dbuf;
}

/**
 * lpfc_complete_unsol_iocb - Complete an unsolicited sequence
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the iocbq struct representing the sequence starting frame.
 * @fch_r_ctl: the r_ctl for the first frame of the sequence.
 * @fch_type: the type for the first frame of the sequence.
 *
 * This function is called with no lock held. This function uses the r_ctl and
 * type of the received sequence to find the correct callback function to call
 * to process the sequence.
 **/
static int
lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                         struct lpfc_iocbq *saveq, uint32_t fch_r_ctl,
                         uint32_t fch_type)
{
        int i;

        /* unSolicited Responses */
        if (pring->prt[0].profile) {
                if (pring->prt[0].lpfc_sli_rcv_unsol_event)
                        (pring->prt[0].lpfc_sli_rcv_unsol_event) (phba, pring,
                                                                        saveq);
                return 1;
        }
        /* We must search, based on rctl / type
           for the right routine */
        for (i = 0; i < pring->num_mask; i++) {
                if ((pring->prt[i].rctl == fch_r_ctl) &&
                    (pring->prt[i].type == fch_type)) {
                        if (pring->prt[i].lpfc_sli_rcv_unsol_event)
                                (pring->prt[i].lpfc_sli_rcv_unsol_event)
                                                (phba, pring, saveq);
                        return 1;
                }
        }
        return 0;
}

/**
 * lpfc_sli_process_unsol_iocb - Unsolicited iocb handler
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the unsolicited iocb.
 *
 * This function is called with no lock held by the ring event handler
 * when there is an unsolicited iocb posted to the response ring by the
 * firmware. This function gets the buffer associated with the iocbs
 * and calls the event handler for the ring. This function handles both
 * qring buffers and hbq buffers.
 * When the function returns 1 the caller can free the iocb object otherwise
 * upper layer functions will free the iocb objects.
 **/
static int
lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                            struct lpfc_iocbq *saveq)
{
        IOCB_t           * irsp;
        WORD5            * w5p;
        uint32_t           Rctl, Type;
        uint32_t           match;
        struct lpfc_iocbq *iocbq;
        struct lpfc_dmabuf *dmzbuf;

        match = 0;
        irsp = &(saveq->iocb);

        if (irsp->ulpCommand == CMD_ASYNC_STATUS) {
                if (pring->lpfc_sli_rcv_async_status)
                        pring->lpfc_sli_rcv_async_status(phba, pring, saveq);
                else
                        lpfc_printf_log(phba,
                                        KERN_WARNING,
                                        LOG_SLI,
                                        "0316 Ring %d handler: unexpected "
                                        "ASYNC_STATUS iocb received evt_code "
                                        "0x%x\n",
                                        pring->ringno,
                                        irsp->un.asyncstat.evt_code);
                return 1;
        }

        if ((irsp->ulpCommand == CMD_IOCB_RET_XRI64_CX) &&
                (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) {
                if (irsp->ulpBdeCount > 0) {
                        dmzbuf = lpfc_sli_get_buff(phba, pring,
                                        irsp->un.ulpWord[3]);
                        lpfc_in_buf_free(phba, dmzbuf);
                }

                if (irsp->ulpBdeCount > 1) {
                        dmzbuf = lpfc_sli_get_buff(phba, pring,
                                        irsp->unsli3.sli3Words[3]);
                        lpfc_in_buf_free(phba, dmzbuf);
                }

                if (irsp->ulpBdeCount > 2) {
                        dmzbuf = lpfc_sli_get_buff(phba, pring,
                                irsp->unsli3.sli3Words[7]);
                        lpfc_in_buf_free(phba, dmzbuf);
                }

                return 1;
        }

        if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) {
                if (irsp->ulpBdeCount != 0) {
                        saveq->context2 = lpfc_sli_get_buff(phba, pring,
                                                irsp->un.ulpWord[3]);
                        if (!saveq->context2)
                                lpfc_printf_log(phba,
                                        KERN_ERR,
                                        LOG_SLI,
                                        "0341 Ring %d Cannot find buffer for "
                                        "an unsolicited iocb. tag 0x%x\n",

                                        pring->ringno,
                                        irsp->un.ulpWord[3]);
                }
                if (irsp->ulpBdeCount == 2) {
                        saveq->context3 = lpfc_sli_get_buff(phba, pring,
                                                irsp->unsli3.sli3Words[7]);
                        if (!saveq->context3)
                                lpfc_printf_log(phba,
                                        KERN_ERR,
                                        LOG_SLI,
                                        "0342 Ring %d Cannot find buffer for an"
                                        " unsolicited iocb. tag 0x%x\n",
                                        pring->ringno,
                                        irsp->unsli3.sli3Words[7]);
                }
                list_for_each_entry(iocbq, &saveq->list, list) {
                        irsp = &(iocbq->iocb);
                        if (irsp->ulpBdeCount != 0) {
                                iocbq->context2 = lpfc_sli_get_buff(phba, pring,
                                                        irsp->un.ulpWord[3]);
                                if (!iocbq->context2)
                                        lpfc_printf_log(phba,
                                                KERN_ERR,
                                                LOG_SLI,
                                                "0343 Ring %d Cannot find "
                                                "buffer for an unsolicited iocb"
                                                ". tag 0x%x\n", pring->ringno,
                                                irsp->un.ulpWord[3]);
                        }
                        if (irsp->ulpBdeCount == 2) {
                                iocbq->context3 = lpfc_sli_get_buff(phba, pring,
                                                irsp->unsli3.sli3Words[7]);
                                if (!iocbq->context3)
                                        lpfc_printf_log(phba,
                                                KERN_ERR,
                                                LOG_SLI,
                                                "0344 Ring %d Cannot find "
                                                "buffer for an unsolicited "
                                                "iocb. tag 0x%x\n",
                                                pring->ringno,
                                                irsp->unsli3.sli3Words[7]);
                        }
                }
        }
        if (irsp->ulpBdeCount != 0 &&
            (irsp->ulpCommand == CMD_IOCB_RCV_CONT64_CX ||
             irsp->ulpStatus == IOSTAT_INTERMED_RSP)) {
                int found = 0;

                /* search continue save q for same XRI */
                list_for_each_entry(iocbq, &pring->iocb_continue_saveq, clist) {
                        if (iocbq->iocb.ulpContext == saveq->iocb.ulpContext) {
                                list_add_tail(&saveq->list, &iocbq->list);
                                found = 1;
                                break;
                        }
                }
                if (!found)
                        list_add_tail(&saveq->clist,
                                      &pring->iocb_continue_saveq);
                if (saveq->iocb.ulpStatus != IOSTAT_INTERMED_RSP) {
                        list_del_init(&iocbq->clist);
                        saveq = iocbq;
                        irsp = &(saveq->iocb);
                } else
                        return 0;
        }
        if ((irsp->ulpCommand == CMD_RCV_ELS_REQ64_CX) ||
            (irsp->ulpCommand == CMD_RCV_ELS_REQ_CX) ||
            (irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX)) {
                Rctl = FC_ELS_REQ;
                Type = FC_ELS_DATA;
        } else {
                w5p = (WORD5 *)&(saveq->iocb.un.ulpWord[5]);
                Rctl = w5p->hcsw.Rctl;
                Type = w5p->hcsw.Type;

                /* Firmware Workaround */
                if ((Rctl == 0) && (pring->ringno == LPFC_ELS_RING) &&
                        (irsp->ulpCommand == CMD_RCV_SEQUENCE64_CX ||
                         irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) {
                        Rctl = FC_ELS_REQ;
                        Type = FC_ELS_DATA;
                        w5p->hcsw.Rctl = Rctl;
                        w5p->hcsw.Type = Type;
                }
        }

        if (!lpfc_complete_unsol_iocb(phba, pring, saveq, Rctl, Type))
                lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
                                "0313 Ring %d handler: unexpected Rctl x%x "
                                "Type x%x received\n",
                                pring->ringno, Rctl, Type);

        return 1;
}

/**
 * lpfc_sli_iocbq_lookup - Find command iocb for the given response iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @prspiocb: Pointer to response iocb object.
 *
 * This function looks up the iocb_lookup table to get the command iocb
 * corresponding to the given response iocb using the iotag of the
 * response iocb. This function is called with the hbalock held.
 * This function returns the command iocb object if it finds the command
 * iocb else returns NULL.
 **/
static struct lpfc_iocbq *
lpfc_sli_iocbq_lookup(struct lpfc_hba *phba,
                      struct lpfc_sli_ring *pring,
                      struct lpfc_iocbq *prspiocb)
{
        struct lpfc_iocbq *cmd_iocb = NULL;
        uint16_t iotag;

        iotag = prspiocb->iocb.ulpIoTag;

        if (iotag != 0 && iotag <= phba->sli.last_iotag) {
                cmd_iocb = phba->sli.iocbq_lookup[iotag];
                list_del_init(&cmd_iocb->list);
                pring->txcmplq_cnt--;
                return cmd_iocb;
        }

        lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                        "0317 iotag x%x is out off "
                        "range: max iotag x%x wd0 x%x\n",
                        iotag, phba->sli.last_iotag,
                        *(((uint32_t *) &prspiocb->iocb) + 7));
        return NULL;
}

/**
 * lpfc_sli_iocbq_lookup_by_tag - Find command iocb for the iotag
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @iotag: IOCB tag.
 *
 * This function looks up the iocb_lookup table to get the command iocb
 * corresponding to the given iotag. This function is called with the
 * hbalock held.
 * This function returns the command iocb object if it finds the command
 * iocb else returns NULL.
 **/
static struct lpfc_iocbq *
lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
                             struct lpfc_sli_ring *pring, uint16_t iotag)
{
        struct lpfc_iocbq *cmd_iocb;

        if (iotag != 0 && iotag <= phba->sli.last_iotag) {
                cmd_iocb = phba->sli.iocbq_lookup[iotag];
                list_del_init(&cmd_iocb->list);
                pring->txcmplq_cnt--;
                return cmd_iocb;
        }

        lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                        "0372 iotag x%x is out off range: max iotag (x%x)\n",
                        iotag, phba->sli.last_iotag);
        return NULL;
}

/**
 * lpfc_sli_process_sol_iocb - process solicited iocb completion
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @saveq: Pointer to the response iocb to be processed.
 *
 * This function is called by the ring event handler for non-fcp
 * rings when there is a new response iocb in the response ring.
 * The caller is not required to hold any locks. This function
 * gets the command iocb associated with the response iocb and
 * calls the completion handler for the command iocb. If there
 * is no completion handler, the function will free the resources
 * associated with command iocb. If the response iocb is for
 * an already aborted command iocb, the status of the completion
 * is changed to IOSTAT_LOCAL_REJECT/IOERR_SLI_ABORTED.
 * This function always returns 1.
 **/
static int
lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                          struct lpfc_iocbq *saveq)
{
        struct lpfc_iocbq *cmdiocbp;
        int rc = 1;
        unsigned long iflag;

        /* Based on the iotag field, get the cmd IOCB from the txcmplq */
        spin_lock_irqsave(&phba->hbalock, iflag);
        cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring, saveq);
        spin_unlock_irqrestore(&phba->hbalock, iflag);

        if (cmdiocbp) {
                if (cmdiocbp->iocb_cmpl) {
                        /*
                         * If an ELS command failed send an event to mgmt
                         * application.
                         */
                        if (saveq->iocb.ulpStatus &&
                             (pring->ringno == LPFC_ELS_RING) &&
                             (cmdiocbp->iocb.ulpCommand ==
                                CMD_ELS_REQUEST64_CR))
                                lpfc_send_els_failure_event(phba,
                                        cmdiocbp, saveq);

                        /*
                         * Post all ELS completions to the worker thread.
                         * All other are passed to the completion callback.
                         */
                        if (pring->ringno == LPFC_ELS_RING) {
                                if (cmdiocbp->iocb_flag & LPFC_DRIVER_ABORTED) {
                                        cmdiocbp->iocb_flag &=
                                                ~LPFC_DRIVER_ABORTED;
                                        saveq->iocb.ulpStatus =
                                                IOSTAT_LOCAL_REJECT;
                                        saveq->iocb.un.ulpWord[4] =
                                                IOERR_SLI_ABORTED;

                                        /* Firmware could still be in progress
                                         * of DMAing payload, so don't free data
                                         * buffer till after a hbeat.
                                         */
                                        saveq->iocb_flag |= LPFC_DELAY_MEM_FREE;
                                }
                        }
                        (cmdiocbp->iocb_cmpl) (phba, cmdiocbp, saveq);
                } else
                        lpfc_sli_release_iocbq(phba, cmdiocbp);
        } else {
                /*
                 * Unknown initiating command based on the response iotag.
                 * This could be the case on the ELS ring because of
                 * lpfc_els_abort().
                 */
                if (pring->ringno != LPFC_ELS_RING) {
                        /*
                         * Ring <ringno> handler: unexpected completion IoTag
                         * <IoTag>
                         */
                        lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
                                         "0322 Ring %d handler: "
                                         "unexpected completion IoTag x%x "
                                         "Data: x%x x%x x%x x%x\n",
                                         pring->ringno,
                                         saveq->iocb.ulpIoTag,
                                         saveq->iocb.ulpStatus,
                                         saveq->iocb.un.ulpWord[4],
                                         saveq->iocb.ulpCommand,
                                         saveq->iocb.ulpContext);
                }
        }

        return rc;
}

/**
 * lpfc_sli_rsp_pointers_error - Response ring pointer error handler
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 *
 * This function is called from the iocb ring event handlers when
 * put pointer is ahead of the get pointer for a ring. This function signal
 * an error attention condition to the worker thread and the worker
 * thread will transition the HBA to offline state.
 **/
static void
lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
{
        struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
        /*
         * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
         * rsp ring <portRspMax>
         */
        lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                        "0312 Ring %d handler: portRspPut %d "
                        "is bigger than rsp ring %d\n",
                        pring->ringno, le32_to_cpu(pgp->rspPutInx),
                        pring->numRiocb);

        phba->link_state = LPFC_HBA_ERROR;

        /*
         * All error attention handlers are posted to
         * worker thread
         */
        phba->work_ha |= HA_ERATT;
        phba->work_hs = HS_FFER3;

        lpfc_worker_wake_up(phba);

        return;
}

/**
 * lpfc_poll_eratt - Error attention polling timer timeout handler
 * @ptr: Pointer to address of HBA context object.
 *
 * This function is invoked by the Error Attention polling timer when the
 * timer times out. It will check the SLI Error Attention register for
 * possible attention events. If so, it will post an Error Attention event
 * and wake up worker thread to process it. Otherwise, it will set up the
 * Error Attention polling timer for the next poll.
 **/
void lpfc_poll_eratt(unsigned long ptr)
{
        struct lpfc_hba *phba;
        uint32_t eratt = 0;

        phba = (struct lpfc_hba *)ptr;

        /* Check chip HA register for error event */
        eratt = lpfc_sli_check_eratt(phba);

        if (eratt)
                /* Tell the worker thread there is work to do */
                lpfc_worker_wake_up(phba);
        else
                /* Restart the timer for next eratt poll */
                mod_timer(&phba->eratt_poll, jiffies +
                                        HZ * LPFC_ERATT_POLL_INTERVAL);
        return;
}

/**
 * lpfc_sli_poll_fcp_ring - Handle FCP ring completion in polling mode
 * @phba: Pointer to HBA context object.
 *
 * This function is called from lpfc_queuecommand, lpfc_poll_timeout,
 * lpfc_abort_handler and lpfc_slave_configure when FCP_RING_POLLING
 * is enabled.
 *
 * The caller does not hold any lock.
 * The function processes each response iocb in the response ring until it
 * finds an iocb with LE bit set and chains all the iocbs upto the iocb with
 * LE bit set. The function will call the completion handler of the command iocb
 * if the response iocb indicates a completion for a command iocb or it is
 * an abort completion.
 **/
void lpfc_sli_poll_fcp_ring(struct lpfc_hba *phba)
{
        struct lpfc_sli      *psli  = &phba->sli;
        struct lpfc_sli_ring *pring = &psli->ring[LPFC_FCP_RING];
        IOCB_t *irsp = NULL;
        IOCB_t *entry = NULL;
        struct lpfc_iocbq *cmdiocbq = NULL;
        struct lpfc_iocbq rspiocbq;
        struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
        uint32_t status;
        uint32_t portRspPut, portRspMax;
        int type;
        uint32_t rsp_cmpl = 0;
        uint32_t ha_copy;
        unsigned long iflags;

        pring->stats.iocb_event++;

        /*
         * The next available response entry should never exceed the maximum
         * entries.  If it does, treat it as an adapter hardware error.
         */
        portRspMax = pring->numRiocb;
        portRspPut = le32_to_cpu(pgp->rspPutInx);
        if (unlikely(portRspPut >= portRspMax)) {
                lpfc_sli_rsp_pointers_error(phba, pring);
                return;
        }

        rmb();
        while (pring->rspidx != portRspPut) {
                entry = lpfc_resp_iocb(phba, pring);
                if (++pring->rspidx >= portRspMax)
                        pring->rspidx = 0;

                lpfc_sli_pcimem_bcopy((uint32_t *) entry,
                                      (uint32_t *) &rspiocbq.iocb,
                                      phba->iocb_rsp_size);
                irsp = &rspiocbq.iocb;
                type = lpfc_sli_iocb_cmd_type(irsp->ulpCommand & CMD_IOCB_MASK);
                pring->stats.iocb_rsp++;
                rsp_cmpl++;

                if (unlikely(irsp->ulpStatus)) {
                        /* Rsp ring <ringno> error: IOCB */
                        lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
                                        "0326 Rsp Ring %d error: IOCB Data: "
                                        "x%x x%x x%x x%x x%x x%x x%x x%x\n",
                                        pring->ringno,
                                        irsp->un.ulpWord[0],
                                        irsp->un.ulpWord[1],
                                        irsp->un.ulpWord[2],
                                        irsp->un.ulpWord[3],
                                        irsp->un.ulpWord[4],
                                        irsp->un.ulpWord[5],
                                        *(uint32_t *)&irsp->un1,
                                        *((uint32_t *)&irsp->un1 + 1));
                }

                switch (type) {
                case LPFC_ABORT_IOCB:
                case LPFC_SOL_IOCB:
                        /*
                         * Idle exchange closed via ABTS from port.  No iocb
                         * resources need to be recovered.
                         */
                        if (unlikely(irsp->ulpCommand == CMD_XRI_ABORTED_CX)) {
                                lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
                                                "0314 IOCB cmd 0x%x "
                                                "processed. Skipping "
                                                "completion",
                                                irsp->ulpCommand);
                                break;
                        }

                        spin_lock_irqsave(&phba->hbalock, iflags);
                        cmdiocbq = lpfc_sli_iocbq_lookup(phba, pring,
                                                         &rspiocbq);
                        spin_unlock_irqrestore(&phba->hbalock, iflags);
                        if ((cmdiocbq) && (cmdiocbq->iocb_cmpl)) {
                                (cmdiocbq->iocb_cmpl)(phba, cmdiocbq,
                                                      &rspiocbq);
                        }
                        break;
                default:
                        if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
                                char adaptermsg[LPFC_MAX_ADPTMSG];
                                memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
                                memcpy(&adaptermsg[0], (uint8_t *) irsp,
                                       MAX_MSG_DATA);
                                dev_warn(&((phba->pcidev)->dev),
                                         "lpfc%d: %s\n",
                                         phba->brd_no, adaptermsg);
                        } else {
                                /* Unknown IOCB command */
                                lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                                                "0321 Unknown IOCB command "
                                                "Data: x%x, x%x x%x x%x x%x\n",
                                                type, irsp->ulpCommand,
                                                irsp->ulpStatus,
                                                irsp->ulpIoTag,
                                                irsp->ulpContext);
                        }
                        break;
                }

                /*
                 * The response IOCB has been processed.  Update the ring
                 * pointer in SLIM.  If the port response put pointer has not
                 * been updated, sync the pgp->rspPutInx and fetch the new port
                 * response put pointer.
                 */
                writel(pring->rspidx, &phba->host_gp[pring->ringno].rspGetInx);

                if (pring->rspidx == portRspPut)
                        portRspPut = le32_to_cpu(pgp->rspPutInx);
        }

        ha_copy = readl(phba->HAregaddr);
        ha_copy >>= (LPFC_FCP_RING * 4);

        if ((rsp_cmpl > 0) && (ha_copy & HA_R0RE_REQ)) {
                spin_lock_irqsave(&phba->hbalock, iflags);
                pring->stats.iocb_rsp_full++;
                status = ((CA_R0ATT | CA_R0RE_RSP) << (LPFC_FCP_RING * 4));
                writel(status, phba->CAregaddr);
                readl(phba->CAregaddr);
                spin_unlock_irqrestore(&phba->hbalock, iflags);
        }
        if ((ha_copy & HA_R0CE_RSP) &&
            (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
                spin_lock_irqsave(&phba->hbalock, iflags);
                pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
                pring->stats.iocb_cmd_empty++;

                /* Force update of the local copy of cmdGetInx */
                pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
                lpfc_sli_resume_iocb(phba, pring);

                if ((pring->lpfc_sli_cmd_available))
                        (pring->lpfc_sli_cmd_available) (phba, pring);

                spin_unlock_irqrestore(&phba->hbalock, iflags);
        }

        return;
}

/**
 * lpfc_sli_handle_fast_ring_event - Handle ring events on FCP ring
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This function is called from the interrupt context when there is a ring
 * event for the fcp ring. The caller does not hold any lock.
 * The function processes each response iocb in the response ring until it
 * finds an iocb with LE bit set and chains all the iocbs upto the iocb with
 * LE bit set. The function will call the completion handler of the command iocb
 * if the response iocb indicates a completion for a command iocb or it is
 * an abort completion. The function will call lpfc_sli_process_unsol_iocb
 * function if this is an unsolicited iocb.
 * This routine presumes LPFC_FCP_RING handling and doesn't bother
 * to check it explicitly. This function always returns 1.
 **/
static int
lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
                                struct lpfc_sli_ring *pring, uint32_t mask)
{
        struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
        IOCB_t *irsp = NULL;
        IOCB_t *entry = NULL;
        struct lpfc_iocbq *cmdiocbq = NULL;
        struct lpfc_iocbq rspiocbq;
        uint32_t status;
        uint32_t portRspPut, portRspMax;
        int rc = 1;
        lpfc_iocb_type type;
        unsigned long iflag;
        uint32_t rsp_cmpl = 0;

        spin_lock_irqsave(&phba->hbalock, iflag);
        pring->stats.iocb_event++;

        /*
         * The next available response entry should never exceed the maximum
         * entries.  If it does, treat it as an adapter hardware error.
         */
        portRspMax = pring->numRiocb;
        portRspPut = le32_to_cpu(pgp->rspPutInx);
        if (unlikely(portRspPut >= portRspMax)) {
                lpfc_sli_rsp_pointers_error(phba, pring);
                spin_unlock_irqrestore(&phba->hbalock, iflag);
                return 1;
        }

        rmb();
        while (pring->rspidx != portRspPut) {
                /*
                 * Fetch an entry off the ring and copy it into a local data
                 * structure.  The copy involves a byte-swap since the
                 * network byte order and pci byte orders are different.
                 */
                entry = lpfc_resp_iocb(phba, pring);
                phba->last_completion_time = jiffies;

                if (++pring->rspidx >= portRspMax)
                        pring->rspidx = 0;

                lpfc_sli_pcimem_bcopy((uint32_t *) entry,
                                      (uint32_t *) &rspiocbq.iocb,
                                      phba->iocb_rsp_size);
                INIT_LIST_HEAD(&(rspiocbq.list));
                irsp = &rspiocbq.iocb;

                type = lpfc_sli_iocb_cmd_type(irsp->ulpCommand & CMD_IOCB_MASK);
                pring->stats.iocb_rsp++;
                rsp_cmpl++;

                if (unlikely(irsp->ulpStatus)) {
                        /*
                         * If resource errors reported from HBA, reduce
                         * queuedepths of the SCSI device.
                         */
                        if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
                                (irsp->un.ulpWord[4] == IOERR_NO_RESOURCES)) {
                                spin_unlock_irqrestore(&phba->hbalock, iflag);
                                phba->lpfc_rampdown_queue_depth(phba);
                                spin_lock_irqsave(&phba->hbalock, iflag);
                        }

                        /* Rsp ring <ringno> error: IOCB */
                        lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
                                        "0336 Rsp Ring %d error: IOCB Data: "
                                        "x%x x%x x%x x%x x%x x%x x%x x%x\n",
                                        pring->ringno,
                                        irsp->un.ulpWord[0],
                                        irsp->un.ulpWord[1],
                                        irsp->un.ulpWord[2],
                                        irsp->un.ulpWord[3],
                                        irsp->un.ulpWord[4],
                                        irsp->un.ulpWord[5],
                                        *(uint32_t *)&irsp->un1,
                                        *((uint32_t *)&irsp->un1 + 1));
                }

                switch (type) {
                case LPFC_ABORT_IOCB:
                case LPFC_SOL_IOCB:
                        /*
                         * Idle exchange closed via ABTS from port.  No iocb
                         * resources need to be recovered.
                         */
                        if (unlikely(irsp->ulpCommand == CMD_XRI_ABORTED_CX)) {
                                lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
                                                "0333 IOCB cmd 0x%x"
                                                " processed. Skipping"
                                                " completion\n",
                                                irsp->ulpCommand);
                                break;
                        }

                        cmdiocbq = lpfc_sli_iocbq_lookup(phba, pring,
                                                         &rspiocbq);
                        if ((cmdiocbq) && (cmdiocbq->iocb_cmpl)) {
                                if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
                                        (cmdiocbq->iocb_cmpl)(phba, cmdiocbq,
                                                              &rspiocbq);
                                } else {
                                        spin_unlock_irqrestore(&phba->hbalock,
                                                               iflag);
                                        (cmdiocbq->iocb_cmpl)(phba, cmdiocbq,
                                                              &rspiocbq);
                                        spin_lock_irqsave(&phba->hbalock,
                                                          iflag);
                                }
                        }
                        break;
                case LPFC_UNSOL_IOCB:
                        spin_unlock_irqrestore(&phba->hbalock, iflag);
                        lpfc_sli_process_unsol_iocb(phba, pring, &rspiocbq);
                        spin_lock_irqsave(&phba->hbalock, iflag);
                        break;
                default:
                        if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
                                char adaptermsg[LPFC_MAX_ADPTMSG];
                                memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
                                memcpy(&adaptermsg[0], (uint8_t *) irsp,
                                       MAX_MSG_DATA);
                                dev_warn(&((phba->pcidev)->dev),
                                         "lpfc%d: %s\n",
                                         phba->brd_no, adaptermsg);
                        } else {
                                /* Unknown IOCB command */
                                lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                                                "0334 Unknown IOCB command "
                                                "Data: x%x, x%x x%x x%x x%x\n",
                                                type, irsp->ulpCommand,
                                                irsp->ulpStatus,
                                                irsp->ulpIoTag,
                                                irsp->ulpContext);
                        }
                        break;
                }

                /*
                 * The response IOCB has been processed.  Update the ring
                 * pointer in SLIM.  If the port response put pointer has not
                 * been updated, sync the pgp->rspPutInx and fetch the new port
                 * response put pointer.
                 */
                writel(pring->rspidx, &phba->host_gp[pring->ringno].rspGetInx);

                if (pring->rspidx == portRspPut)
                        portRspPut = le32_to_cpu(pgp->rspPutInx);
        }

        if ((rsp_cmpl > 0) && (mask & HA_R0RE_REQ)) {
                pring->stats.iocb_rsp_full++;
                status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
                writel(status, phba->CAregaddr);
                readl(phba->CAregaddr);
        }
        if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
                pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
                pring->stats.iocb_cmd_empty++;

                /* Force update of the local copy of cmdGetInx */
                pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
                lpfc_sli_resume_iocb(phba, pring);

                if ((pring->lpfc_sli_cmd_available))
                        (pring->lpfc_sli_cmd_available) (phba, pring);

        }

        spin_unlock_irqrestore(&phba->hbalock, iflag);
        return rc;
}

/**
 * lpfc_sli_sp_handle_rspiocb - Handle slow-path response iocb
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @rspiocbp: Pointer to driver response IOCB object.
 *
 * This function is called from the worker thread when there is a slow-path
 * response IOCB to process. This function chains all the response iocbs until
 * seeing the iocb with the LE bit set. The function will call
 * lpfc_sli_process_sol_iocb function if the response iocb indicates a
 * completion of a command iocb. The function will call the
 * lpfc_sli_process_unsol_iocb function if this is an unsolicited iocb.
 * The function frees the resources or calls the completion handler if this
 * iocb is an abort completion. The function returns NULL when the response
 * iocb has the LE bit set and all the chained iocbs are processed, otherwise
 * this function shall chain the iocb on to the iocb_continueq and return the
 * response iocb passed in.
 **/
static struct lpfc_iocbq *
lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
                        struct lpfc_iocbq *rspiocbp)
{
        struct lpfc_iocbq *saveq;
        struct lpfc_iocbq *cmdiocbp;
        struct lpfc_iocbq *next_iocb;
        IOCB_t *irsp = NULL;
        uint32_t free_saveq;
        uint8_t iocb_cmd_type;
        lpfc_iocb_type type;
        unsigned long iflag;
        int rc;

        spin_lock_irqsave(&phba->hbalock, iflag);
        /* First add the response iocb to the countinueq list */
        list_add_tail(&rspiocbp->list, &(pring->iocb_continueq));
        pring->iocb_continueq_cnt++;

        /* Now, determine whetehr the list is completed for processing */
        irsp = &rspiocbp->iocb;
        if (irsp->ulpLe) {
                /*
                 * By default, the driver expects to free all resources
                 * associated with this iocb completion.
                 */
                free_saveq = 1;
                saveq = list_get_first(&pring->iocb_continueq,
                                       struct lpfc_iocbq, list);
                irsp = &(saveq->iocb);
                list_del_init(&pring->iocb_continueq);
                pring->iocb_continueq_cnt = 0;

                pring->stats.iocb_rsp++;

                /*
                 * If resource errors reported from HBA, reduce
                 * queuedepths of the SCSI device.
                 */
                if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
                    (irsp->un.ulpWord[4] == IOERR_NO_RESOURCES)) {
                        spin_unlock_irqrestore(&phba->hbalock, iflag);
                        phba->lpfc_rampdown_queue_depth(phba);
                        spin_lock_irqsave(&phba->hbalock, iflag);
                }

                if (irsp->ulpStatus) {
                        /* Rsp ring <ringno> error: IOCB */
                        lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
                                        "0328 Rsp Ring %d error: "
                                        "IOCB Data: "
                                        "x%x x%x x%x x%x "
                                        "x%x x%x x%x x%x "
                                        "x%x x%x x%x x%x "
                                        "x%x x%x x%x x%x\n",
                                        pring->ringno,
                                        irsp->un.ulpWord[0],
                                        irsp->un.ulpWord[1],
                                        irsp->un.ulpWord[2],
                                        irsp->un.ulpWord[3],
                                        irsp->un.ulpWord[4],
                                        irsp->un.ulpWord[5],
                                        *(((uint32_t *) irsp) + 6),
                                        *(((uint32_t *) irsp) + 7),
                                        *(((uint32_t *) irsp) + 8),
                                        *(((uint32_t *) irsp) + 9),
                                        *(((uint32_t *) irsp) + 10),
                                        *(((uint32_t *) irsp) + 11),
                                        *(((uint32_t *) irsp) + 12),
                                        *(((uint32_t *) irsp) + 13),
                                        *(((uint32_t *) irsp) + 14),
                                        *(((uint32_t *) irsp) + 15));
                }

                /*
                 * Fetch the IOCB command type and call the correct completion
                 * routine. Solicited and Unsolicited IOCBs on the ELS ring
                 * get freed back to the lpfc_iocb_list by the discovery
                 * kernel thread.
                 */
                iocb_cmd_type = irsp->ulpCommand & CMD_IOCB_MASK;
                type = lpfc_sli_iocb_cmd_type(iocb_cmd_type);
                switch (type) {
                case LPFC_SOL_IOCB:
                        spin_unlock_irqrestore(&phba->hbalock, iflag);
                        rc = lpfc_sli_process_sol_iocb(phba, pring, saveq);
                        spin_lock_irqsave(&phba->hbalock, iflag);
                        break;

                case LPFC_UNSOL_IOCB:
                        spin_unlock_irqrestore(&phba->hbalock, iflag);
                        rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq);
                        spin_lock_irqsave(&phba->hbalock, iflag);
                        if (!rc)
                                free_saveq = 0;
                        break;

                case LPFC_ABORT_IOCB:
                        cmdiocbp = NULL;
                        if (irsp->ulpCommand != CMD_XRI_ABORTED_CX)
                                cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring,
                                                                 saveq);
                        if (cmdiocbp) {
                                /* Call the specified completion routine */
                                if (cmdiocbp->iocb_cmpl) {
                                        spin_unlock_irqrestore(&phba->hbalock,
                                                               iflag);
                                        (cmdiocbp->iocb_cmpl)(phba, cmdiocbp,
                                                              saveq);
                                        spin_lock_irqsave(&phba->hbalock,
                                                          iflag);
                                } else
                                        __lpfc_sli_release_iocbq(phba,
                                                                 cmdiocbp);
                        }
                        break;

                case LPFC_UNKNOWN_IOCB:
                        if (irsp->ulpCommand == CMD_ADAPTER_MSG) {
                                char adaptermsg[LPFC_MAX_ADPTMSG];
                                memset(adaptermsg, 0, LPFC_MAX_ADPTMSG);
                                memcpy(&adaptermsg[0], (uint8_t *)irsp,
                                       MAX_MSG_DATA);
                                dev_warn(&((phba->pcidev)->dev),
                                         "lpfc%d: %s\n",
                                         phba->brd_no, adaptermsg);
                        } else {
                                /* Unknown IOCB command */
                                lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                                                "0335 Unknown IOCB "
                                                "command Data: x%x "
                                                "x%x x%x x%x\n",
                                                irsp->ulpCommand,
                                                irsp->ulpStatus,
                                                irsp->ulpIoTag,
                                                irsp->ulpContext);
                        }
                        break;
                }

                if (free_saveq) {
                        list_for_each_entry_safe(rspiocbp, next_iocb,
                                                 &saveq->list, list) {
                                list_del(&rspiocbp->list);
                                __lpfc_sli_release_iocbq(phba, rspiocbp);
                        }
                        __lpfc_sli_release_iocbq(phba, saveq);
                }
                rspiocbp = NULL;
        }
        spin_unlock_irqrestore(&phba->hbalock, iflag);
        return rspiocbp;
}

/**
 * lpfc_sli_handle_slow_ring_event - Wrapper func for handling slow-path iocbs
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This routine wraps the actual slow_ring event process routine from the
 * API jump table function pointer from the lpfc_hba struct.
 **/
void
lpfc_sli_handle_slow_ring_event(struct lpfc_hba *phba,
                                struct lpfc_sli_ring *pring, uint32_t mask)
{
        phba->lpfc_sli_handle_slow_ring_event(phba, pring, mask);
}

/**
 * lpfc_sli_handle_slow_ring_event_s3 - Handle SLI3 ring event for non-FCP rings
 * @phba: Pointer to HBA context object.
 * @pring: Pointer to driver SLI ring object.
 * @mask: Host attention register mask for this ring.
 *
 * This function is called from the worker thread when there is a ring event
 * for non-fcp rings. The caller does not hold any lock. The function will
 * remove each response iocb in the response ring and calls the handle
 * response iocb routine (lpfc_sli_sp_handle_rspiocb) to process it.
 **/
static void
lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
                                   struct lpfc_sli_ring *pring, uint32_t mask)
{
        struct lpfc_pgp *pgp;
        IOCB_t *entry;
        IOCB_t *irsp = NULL;
        struct lpfc_iocbq *rspiocbp = NULL;
        uint32_t portRspPut, portRspMax;
        unsigned long iflag;
        uint32_t status;

        pgp = &phba->port_gp[pring->ringno];
        spin_lock_irqsave(&phba->hbalock, iflag);
        pring->stats.iocb_event++;

        /*
         * The next available response entry should never exceed the maximum
         * entries.  If it does, treat it as an adapter hardware error.
         */
        portRspMax = pring->numRiocb;
        portRspPut = le32_to_cpu(pgp->rspPutInx);
        if (portRspPut >= portRspMax) {
                /*
                 * Ring <ringno> handler: portRspPut <portRspPut> is bigger than
                 * rsp ring <portRspMax>
                 */
                lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
                                "0303 Ring %d handler: portRspPut %d "
                                "is bigger than rsp ring %d\n",
                                pring->ringno, portRspPut, portRspMax);

                phba->link_state = LPFC_HBA_ERROR;
                spin_unlock_irqrestore(&phba->hbalock, iflag);

                phba->work_hs = HS_FFER3;
                lpfc_handle_eratt(phba);

                return;
        }

        rmb();
        while (pring->rspidx != portRspPut) {
                /*
                 * Build a completion list and call the appropriate handler.
                 * The process is to get the next available response iocb, get
                 * a free iocb from the list, copy the response data into the
                 * free iocb, insert to the continuation list, and update the
                 * next response index to slim.  This process makes response
                 * iocb's in the ring available to DMA as fast as possible but
                 * pays a penalty for a copy operation.  Since the iocb is
                 * only 32 bytes, this penalty is considered small relative to
                 * the PCI reads for register values and a slim write.  When
                 * the ulpLe field is set, the entire Command has been
                 * received.
                 */
                entry = lpfc_resp_iocb(phba, pring);

                phba->last_completion_time = jiffies;
                rspiocbp = __lpfc_sli_get_iocbq(phba);
                if (rspiocbp == NULL) {
                        printk(KERN_ERR "%s: out of buffers! Failing "
                               "completion.\n", __func__);
                        break;
                }

                lpfc_sli_pcimem_bcopy(entry, &rspiocbp->iocb,
                                      phba->iocb_rsp_size);
                irsp = &rspiocbp->iocb;

                if (++pring->rspidx >= portRspMax)
                        pring->rspidx = 0;

                if (pring->ringno == LPFC_ELS_RING) {
                        lpfc_debugfs_slow_ring_trc(phba,
                        "IOCB rsp ring:   wd4:x%08x wd6:x%08x wd7:x%08x",
                                *(((uint32_t *) irsp) + 4),
                                *(((uint32_t *) irsp) + 6),
                                *(((uint32_t *) irsp) + 7));
                }

                writel(pring->rspidx, &phba->host_gp[pring->ringno].rspGetInx);

                spin_unlock_irqrestore(&phba->hbalock, iflag);
                /* Handle the response IOCB */
                rspiocbp = lpfc_sli_sp_handle_rspiocb(phba, pring, rspiocbp);
                spin_lock_irqsave(&phba->hbalock, iflag);

                /*
                 * If the port response put pointer has not been updated, sync
                 * the pgp->rspPutInx in the MAILBOX_tand fetch the new port
                 * response put pointer.
                 */
                if (pring->rspidx == portRspPut) {
                        portRspPut = le32_to_cpu(pgp->rspPutInx);
                }
        } /* while (pring->rspidx != portRspPut) */

        if ((rspiocbp != NULL) && (mask & HA_R0RE_REQ)) {
                /* At least one response entry has been freed */
                pring->stats.iocb_rsp_full++;
                /* SET RxRE_RSP in Chip Att register */
                status = ((CA_R0ATT | CA_R0RE_RSP) << (pring->ringno * 4));
                writel(status, phba->CAregaddr);
                readl(phba->CAregaddr); /* flush */
        }
        if ((mask & HA_R0CE_RSP) && (pring->flag & LPFC_CALL_RING_AVAILABLE)) {
                pring->flag &= ~LPFC_CALL_RING_AVAILABLE;
                pring->stats.iocb_cmd_empty++;

                /* Force update of the local copy of cmdGetInx */
                pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
                lpfc_sli_resume_iocb(phba, pring);

                if ((pring->lpfc_sli_cmd_available))
                        (pring->lpfc_sli_cmd_available) (phba, pring);

        }

        spin_unlock_irqrestore(&phba->hbalock, iflag);