/*******************************************************************************
 *
 * Intel Ethernet Controller XL710 Family Linux Driver
 * Copyright(c) 2013 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Contact Information:
 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 ******************************************************************************/

/* Local includes */
#include "i40e.h"

const char i40e_driver_name[] = "i40e";
static const char i40e_driver_string[] =
                        "Intel(R) Ethernet Connection XL710 Network Driver";

#define DRV_KERN "-k"

#define DRV_VERSION_MAJOR 0
#define DRV_VERSION_MINOR 3
#define DRV_VERSION_BUILD 11
#define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
             __stringify(DRV_VERSION_MINOR) "." \
             __stringify(DRV_VERSION_BUILD)    DRV_KERN
const char i40e_driver_version_str[] = DRV_VERSION;
static const char i40e_copyright[] = "Copyright (c) 2013 Intel Corporation.";

/* a bit of forward declarations */
static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
static void i40e_handle_reset_warning(struct i40e_pf *pf);
static int i40e_add_vsi(struct i40e_vsi *vsi);
static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
static int i40e_setup_pf_switch(struct i40e_pf *pf);
static int i40e_setup_misc_vector(struct i40e_pf *pf);
static void i40e_determine_queue_usage(struct i40e_pf *pf);
static int i40e_setup_pf_filter_control(struct i40e_pf *pf);

/* i40e_pci_tbl - PCI Device ID Table
 *
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = {
        {PCI_VDEVICE(INTEL, I40E_SFP_XL710_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_SFP_X710_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_QEMU_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_KX_A_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_KX_B_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_KX_C_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_KX_D_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_QSFP_A_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_QSFP_B_DEVICE_ID), 0},
        {PCI_VDEVICE(INTEL, I40E_QSFP_C_DEVICE_ID), 0},
        /* required last entry */
        {0, }
};
MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);

#define I40E_MAX_VF_COUNT 128
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

/**
 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 * @hw:   pointer to the HW structure
 * @mem:  ptr to mem struct to fill out
 * @size: size of memory requested
 * @alignment: what to align the allocation to
 **/
int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
                            u64 size, u32 alignment)
{
        struct i40e_pf *pf = (struct i40e_pf *)hw->back;

        mem->size = ALIGN(size, alignment);
        mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
                                      &mem->pa, GFP_KERNEL);
        if (!mem->va)
                return -ENOMEM;

        return 0;
}

/**
 * i40e_free_dma_mem_d - OS specific memory free for shared code
 * @hw:   pointer to the HW structure
 * @mem:  ptr to mem struct to free
 **/
int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
{
        struct i40e_pf *pf = (struct i40e_pf *)hw->back;

        dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
        mem->va = NULL;
        mem->pa = 0;
        mem->size = 0;

        return 0;
}

/**
 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 * @hw:   pointer to the HW structure
 * @mem:  ptr to mem struct to fill out
 * @size: size of memory requested
 **/
int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
                             u32 size)
{
        mem->size = size;
        mem->va = kzalloc(size, GFP_KERNEL);

        if (!mem->va)
                return -ENOMEM;

        return 0;
}

/**
 * i40e_free_virt_mem_d - OS specific memory free for shared code
 * @hw:   pointer to the HW structure
 * @mem:  ptr to mem struct to free
 **/
int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
{
        /* it's ok to kfree a NULL pointer */
        kfree(mem->va);
        mem->va = NULL;
        mem->size = 0;

        return 0;
}

/**
 * i40e_get_lump - find a lump of free generic resource
 * @pf: board private structure
 * @pile: the pile of resource to search
 * @needed: the number of items needed
 * @id: an owner id to stick on the items assigned
 *
 * Returns the base item index of the lump, or negative for error
 *
 * The search_hint trick and lack of advanced fit-finding only work
 * because we're highly likely to have all the same size lump requests.
 * Linear search time and any fragmentation should be minimal.
 **/
static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
                         u16 needed, u16 id)
{
        int ret = -ENOMEM;
        int i, j;

        if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
                dev_info(&pf->pdev->dev,
                         "param err: pile=%p needed=%d id=0x%04x\n",
                         pile, needed, id);
                return -EINVAL;
        }

        /* start the linear search with an imperfect hint */
        i = pile->search_hint;
        while (i < pile->num_entries) {
                /* skip already allocated entries */
                if (pile->list[i] & I40E_PILE_VALID_BIT) {
                        i++;
                        continue;
                }

                /* do we have enough in this lump? */
                for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
                        if (pile->list[i+j] & I40E_PILE_VALID_BIT)
                                break;
                }

                if (j == needed) {
                        /* there was enough, so assign it to the requestor */
                        for (j = 0; j < needed; j++)
                                pile->list[i+j] = id | I40E_PILE_VALID_BIT;
                        ret = i;
                        pile->search_hint = i + j;
                        break;
                } else {
                        /* not enough, so skip over it and continue looking */
                        i += j;
                }
        }

        return ret;
}

/**
 * i40e_put_lump - return a lump of generic resource
 * @pile: the pile of resource to search
 * @index: the base item index
 * @id: the owner id of the items assigned
 *
 * Returns the count of items in the lump
 **/
static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
{
        int valid_id = (id | I40E_PILE_VALID_BIT);
        int count = 0;
        int i;

        if (!pile || index >= pile->num_entries)
                return -EINVAL;

        for (i = index;
             i < pile->num_entries && pile->list[i] == valid_id;
             i++) {
                pile->list[i] = 0;
                count++;
        }

        if (count && index < pile->search_hint)
                pile->search_hint = index;

        return count;
}

/**
 * i40e_service_event_schedule - Schedule the service task to wake up
 * @pf: board private structure
 *
 * If not already scheduled, this puts the task into the work queue
 **/
static void i40e_service_event_schedule(struct i40e_pf *pf)
{
        if (!test_bit(__I40E_DOWN, &pf->state) &&
            !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
            !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
                schedule_work(&pf->service_task);
}

/**
 * i40e_tx_timeout - Respond to a Tx Hang
 * @netdev: network interface device structure
 *
 * If any port has noticed a Tx timeout, it is likely that the whole
 * device is munged, not just the one netdev port, so go for the full
 * reset.
 **/
static void i40e_tx_timeout(struct net_device *netdev)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;
        struct i40e_pf *pf = vsi->back;

        pf->tx_timeout_count++;

        if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
                pf->tx_timeout_recovery_level = 0;
        pf->tx_timeout_last_recovery = jiffies;
        netdev_info(netdev, "tx_timeout recovery level %d\n",
                    pf->tx_timeout_recovery_level);

        switch (pf->tx_timeout_recovery_level) {
        case 0:
                /* disable and re-enable queues for the VSI */
                if (in_interrupt()) {
                        set_bit(__I40E_REINIT_REQUESTED, &pf->state);
                        set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
                } else {
                        i40e_vsi_reinit_locked(vsi);
                }
                break;
        case 1:
                set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
                break;
        case 2:
                set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
                break;
        case 3:
                set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
                break;
        default:
                netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
                i40e_down(vsi);
                break;
        }
        i40e_service_event_schedule(pf);
        pf->tx_timeout_recovery_level++;
}

/**
 * i40e_release_rx_desc - Store the new tail and head values
 * @rx_ring: ring to bump
 * @val: new head index
 **/
static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
{
        rx_ring->next_to_use = val;

        /* Force memory writes to complete before letting h/w
         * know there are new descriptors to fetch.  (Only
         * applicable for weak-ordered memory model archs,
         * such as IA-64).
         */
        wmb();
        writel(val, rx_ring->tail);
}

/**
 * i40e_get_vsi_stats_struct - Get System Network Statistics
 * @vsi: the VSI we care about
 *
 * Returns the address of the device statistics structure.
 * The statistics are actually updated from the service task.
 **/
struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
{
        return &vsi->net_stats;
}

/**
 * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 * @netdev: network interface device structure
 *
 * Returns the address of the device statistics structure.
 * The statistics are actually updated from the service task.
 **/
static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
                                             struct net_device *netdev,
                                             struct rtnl_link_stats64 *stats)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;
        struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
        int i;

        if (!vsi->tx_rings)
                return stats;

        rcu_read_lock();
        for (i = 0; i < vsi->num_queue_pairs; i++) {
                struct i40e_ring *tx_ring, *rx_ring;
                u64 bytes, packets;
                unsigned int start;

                tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
                if (!tx_ring)
                        continue;

                do {
                        start = u64_stats_fetch_begin_bh(&tx_ring->syncp);
                        packets = tx_ring->stats.packets;
                        bytes   = tx_ring->stats.bytes;
                } while (u64_stats_fetch_retry_bh(&tx_ring->syncp, start));

                stats->tx_packets += packets;
                stats->tx_bytes   += bytes;
                rx_ring = &tx_ring[1];

                do {
                        start = u64_stats_fetch_begin_bh(&rx_ring->syncp);
                        packets = rx_ring->stats.packets;
                        bytes   = rx_ring->stats.bytes;
                } while (u64_stats_fetch_retry_bh(&rx_ring->syncp, start));

                stats->rx_packets += packets;
                stats->rx_bytes   += bytes;
        }
        rcu_read_unlock();

        /* following stats updated by ixgbe_watchdog_task() */
        stats->multicast        = vsi_stats->multicast;
        stats->tx_errors        = vsi_stats->tx_errors;
        stats->tx_dropped       = vsi_stats->tx_dropped;
        stats->rx_errors        = vsi_stats->rx_errors;
        stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
        stats->rx_length_errors = vsi_stats->rx_length_errors;

        return stats;
}

/**
 * i40e_vsi_reset_stats - Resets all stats of the given vsi
 * @vsi: the VSI to have its stats reset
 **/
void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
{
        struct rtnl_link_stats64 *ns;
        int i;

        if (!vsi)
                return;

        ns = i40e_get_vsi_stats_struct(vsi);
        memset(ns, 0, sizeof(*ns));
        memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
        memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
        memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
        if (vsi->rx_rings)
                for (i = 0; i < vsi->num_queue_pairs; i++) {
                        memset(&vsi->rx_rings[i]->stats, 0 ,
                               sizeof(vsi->rx_rings[i]->stats));
                        memset(&vsi->rx_rings[i]->rx_stats, 0 ,
                               sizeof(vsi->rx_rings[i]->rx_stats));
                        memset(&vsi->tx_rings[i]->stats, 0 ,
                               sizeof(vsi->tx_rings[i]->stats));
                        memset(&vsi->tx_rings[i]->tx_stats, 0,
                               sizeof(vsi->tx_rings[i]->tx_stats));
                }
        vsi->stat_offsets_loaded = false;
}

/**
 * i40e_pf_reset_stats - Reset all of the stats for the given pf
 * @pf: the PF to be reset
 **/
void i40e_pf_reset_stats(struct i40e_pf *pf)
{
        memset(&pf->stats, 0, sizeof(pf->stats));
        memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
        pf->stat_offsets_loaded = false;
}

/**
 * i40e_stat_update48 - read and update a 48 bit stat from the chip
 * @hw: ptr to the hardware info
 * @hireg: the high 32 bit reg to read
 * @loreg: the low 32 bit reg to read
 * @offset_loaded: has the initial offset been loaded yet
 * @offset: ptr to current offset value
 * @stat: ptr to the stat
 *
 * Since the device stats are not reset at PFReset, they likely will not
 * be zeroed when the driver starts.  We'll save the first values read
 * and use them as offsets to be subtracted from the raw values in order
 * to report stats that count from zero.  In the process, we also manage
 * the potential roll-over.
 **/
static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
                               bool offset_loaded, u64 *offset, u64 *stat)
{
        u64 new_data;

        if (hw->device_id == I40E_QEMU_DEVICE_ID) {
                new_data = rd32(hw, loreg);
                new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
        } else {
                new_data = rd64(hw, loreg);
        }
        if (!offset_loaded)
                *offset = new_data;
        if (likely(new_data >= *offset))
                *stat = new_data - *offset;
        else
                *stat = (new_data + ((u64)1 << 48)) - *offset;
        *stat &= 0xFFFFFFFFFFFFULL;
}

/**
 * i40e_stat_update32 - read and update a 32 bit stat from the chip
 * @hw: ptr to the hardware info
 * @reg: the hw reg to read
 * @offset_loaded: has the initial offset been loaded yet
 * @offset: ptr to current offset value
 * @stat: ptr to the stat
 **/
static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
                               bool offset_loaded, u64 *offset, u64 *stat)
{
        u32 new_data;

        new_data = rd32(hw, reg);
        if (!offset_loaded)
                *offset = new_data;
        if (likely(new_data >= *offset))
                *stat = (u32)(new_data - *offset);
        else
                *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
}

/**
 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 * @vsi: the VSI to be updated
 **/
void i40e_update_eth_stats(struct i40e_vsi *vsi)
{
        int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        struct i40e_eth_stats *oes;
        struct i40e_eth_stats *es;     /* device's eth stats */

        es = &vsi->eth_stats;
        oes = &vsi->eth_stats_offsets;

        /* Gather up the stats that the hw collects */
        i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->tx_errors, &es->tx_errors);
        i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->rx_discards, &es->rx_discards);

        i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
                           I40E_GLV_GORCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->rx_bytes, &es->rx_bytes);
        i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
                           I40E_GLV_UPRCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->rx_unicast, &es->rx_unicast);
        i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
                           I40E_GLV_MPRCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->rx_multicast, &es->rx_multicast);
        i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
                           I40E_GLV_BPRCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->rx_broadcast, &es->rx_broadcast);

        i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
                           I40E_GLV_GOTCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->tx_bytes, &es->tx_bytes);
        i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
                           I40E_GLV_UPTCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->tx_unicast, &es->tx_unicast);
        i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
                           I40E_GLV_MPTCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->tx_multicast, &es->tx_multicast);
        i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
                           I40E_GLV_BPTCL(stat_idx),
                           vsi->stat_offsets_loaded,
                           &oes->tx_broadcast, &es->tx_broadcast);
        vsi->stat_offsets_loaded = true;
}

/**
 * i40e_update_veb_stats - Update Switch component statistics
 * @veb: the VEB being updated
 **/
static void i40e_update_veb_stats(struct i40e_veb *veb)
{
        struct i40e_pf *pf = veb->pf;
        struct i40e_hw *hw = &pf->hw;
        struct i40e_eth_stats *oes;
        struct i40e_eth_stats *es;     /* device's eth stats */
        int idx = 0;

        idx = veb->stats_idx;
        es = &veb->stats;
        oes = &veb->stats_offsets;

        /* Gather up the stats that the hw collects */
        i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
                           veb->stat_offsets_loaded,
                           &oes->tx_discards, &es->tx_discards);
        i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
                           veb->stat_offsets_loaded,
                           &oes->rx_unknown_protocol, &es->rx_unknown_protocol);

        i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->rx_bytes, &es->rx_bytes);
        i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->rx_unicast, &es->rx_unicast);
        i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->rx_multicast, &es->rx_multicast);
        i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->rx_broadcast, &es->rx_broadcast);

        i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->tx_bytes, &es->tx_bytes);
        i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->tx_unicast, &es->tx_unicast);
        i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->tx_multicast, &es->tx_multicast);
        i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
                           veb->stat_offsets_loaded,
                           &oes->tx_broadcast, &es->tx_broadcast);
        veb->stat_offsets_loaded = true;
}

/**
 * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 * @pf: the corresponding PF
 *
 * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 **/
static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
{
        struct i40e_hw_port_stats *osd = &pf->stats_offsets;
        struct i40e_hw_port_stats *nsd = &pf->stats;
        struct i40e_hw *hw = &pf->hw;
        u64 xoff = 0;
        u16 i, v;

        if ((hw->fc.current_mode != I40E_FC_FULL) &&
            (hw->fc.current_mode != I40E_FC_RX_PAUSE))
                return;

        xoff = nsd->link_xoff_rx;
        i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
                           pf->stat_offsets_loaded,
                           &osd->link_xoff_rx, &nsd->link_xoff_rx);

        /* No new LFC xoff rx */
        if (!(nsd->link_xoff_rx - xoff))
                return;

        /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
        for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
                struct i40e_vsi *vsi = pf->vsi[v];

                if (!vsi)
                        continue;

                for (i = 0; i < vsi->num_queue_pairs; i++) {
                        struct i40e_ring *ring = vsi->tx_rings[i];
                        clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
                }
        }
}

/**
 * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 * @pf: the corresponding PF
 *
 * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 **/
static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
{
        struct i40e_hw_port_stats *osd = &pf->stats_offsets;
        struct i40e_hw_port_stats *nsd = &pf->stats;
        bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
        struct i40e_dcbx_config *dcb_cfg;
        struct i40e_hw *hw = &pf->hw;
        u16 i, v;
        u8 tc;

        dcb_cfg = &hw->local_dcbx_config;

        /* See if DCB enabled with PFC TC */
        if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
            !(dcb_cfg->pfc.pfcenable)) {
                i40e_update_link_xoff_rx(pf);
                return;
        }

        for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
                u64 prio_xoff = nsd->priority_xoff_rx[i];
                i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
                                   pf->stat_offsets_loaded,
                                   &osd->priority_xoff_rx[i],
                                   &nsd->priority_xoff_rx[i]);

                /* No new PFC xoff rx */
                if (!(nsd->priority_xoff_rx[i] - prio_xoff))
                        continue;
                /* Get the TC for given priority */
                tc = dcb_cfg->etscfg.prioritytable[i];
                xoff[tc] = true;
        }

        /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
        for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
                struct i40e_vsi *vsi = pf->vsi[v];

                if (!vsi)
                        continue;

                for (i = 0; i < vsi->num_queue_pairs; i++) {
                        struct i40e_ring *ring = vsi->tx_rings[i];

                        tc = ring->dcb_tc;
                        if (xoff[tc])
                                clear_bit(__I40E_HANG_CHECK_ARMED,
                                          &ring->state);
                }
        }
}

/**
 * i40e_update_stats - Update the board statistics counters.
 * @vsi: the VSI to be updated
 *
 * There are a few instances where we store the same stat in a
 * couple of different structs.  This is partly because we have
 * the netdev stats that need to be filled out, which is slightly
 * different from the "eth_stats" defined by the chip and used in
 * VF communications.  We sort it all out here in a central place.
 **/
void i40e_update_stats(struct i40e_vsi *vsi)
{
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        struct rtnl_link_stats64 *ons;
        struct rtnl_link_stats64 *ns;   /* netdev stats */
        struct i40e_eth_stats *oes;
        struct i40e_eth_stats *es;     /* device's eth stats */
        u32 tx_restart, tx_busy;
        u32 rx_page, rx_buf;
        u64 rx_p, rx_b;
        u64 tx_p, tx_b;
        int i;
        u16 q;

        if (test_bit(__I40E_DOWN, &vsi->state) ||
            test_bit(__I40E_CONFIG_BUSY, &pf->state))
                return;

        ns = i40e_get_vsi_stats_struct(vsi);
        ons = &vsi->net_stats_offsets;
        es = &vsi->eth_stats;
        oes = &vsi->eth_stats_offsets;

        /* Gather up the netdev and vsi stats that the driver collects
         * on the fly during packet processing
         */
        rx_b = rx_p = 0;
        tx_b = tx_p = 0;
        tx_restart = tx_busy = 0;
        rx_page = 0;
        rx_buf = 0;
        rcu_read_lock();
        for (q = 0; q < vsi->num_queue_pairs; q++) {
                struct i40e_ring *p;
                u64 bytes, packets;
                unsigned int start;

                /* locate Tx ring */
                p = ACCESS_ONCE(vsi->tx_rings[q]);

                do {
                        start = u64_stats_fetch_begin_bh(&p->syncp);
                        packets = p->stats.packets;
                        bytes = p->stats.bytes;
                } while (u64_stats_fetch_retry_bh(&p->syncp, start));
                tx_b += bytes;
                tx_p += packets;
                tx_restart += p->tx_stats.restart_queue;
                tx_busy += p->tx_stats.tx_busy;

                /* Rx queue is part of the same block as Tx queue */
                p = &p[1];
                do {
                        start = u64_stats_fetch_begin_bh(&p->syncp);
                        packets = p->stats.packets;
                        bytes = p->stats.bytes;
                } while (u64_stats_fetch_retry_bh(&p->syncp, start));
                rx_b += bytes;
                rx_p += packets;
                rx_buf += p->rx_stats.alloc_rx_buff_failed;
                rx_page += p->rx_stats.alloc_rx_page_failed;
        }
        rcu_read_unlock();
        vsi->tx_restart = tx_restart;
        vsi->tx_busy = tx_busy;
        vsi->rx_page_failed = rx_page;
        vsi->rx_buf_failed = rx_buf;

        ns->rx_packets = rx_p;
        ns->rx_bytes = rx_b;
        ns->tx_packets = tx_p;
        ns->tx_bytes = tx_b;

        i40e_update_eth_stats(vsi);
        /* update netdev stats from eth stats */
        ons->rx_errors = oes->rx_errors;
        ns->rx_errors = es->rx_errors;
        ons->tx_errors = oes->tx_errors;
        ns->tx_errors = es->tx_errors;
        ons->multicast = oes->rx_multicast;
        ns->multicast = es->rx_multicast;
        ons->tx_dropped = oes->tx_discards;
        ns->tx_dropped = es->tx_discards;

        /* Get the port data only if this is the main PF VSI */
        if (vsi == pf->vsi[pf->lan_vsi]) {
                struct i40e_hw_port_stats *nsd = &pf->stats;
                struct i40e_hw_port_stats *osd = &pf->stats_offsets;

                i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
                                   I40E_GLPRT_GORCL(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
                i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
                                   I40E_GLPRT_GOTCL(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
                i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->eth.rx_discards,
                                   &nsd->eth.rx_discards);
                i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->eth.tx_discards,
                                   &nsd->eth.tx_discards);
                i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
                                   I40E_GLPRT_MPRCL(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->eth.rx_multicast,
                                   &nsd->eth.rx_multicast);

                i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_dropped_link_down,
                                   &nsd->tx_dropped_link_down);

                i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->crc_errors, &nsd->crc_errors);
                ns->rx_crc_errors = nsd->crc_errors;

                i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->illegal_bytes, &nsd->illegal_bytes);
                ns->rx_errors = nsd->crc_errors
                                + nsd->illegal_bytes;

                i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->mac_local_faults,
                                   &nsd->mac_local_faults);
                i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->mac_remote_faults,
                                   &nsd->mac_remote_faults);

                i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_length_errors,
                                   &nsd->rx_length_errors);
                ns->rx_length_errors = nsd->rx_length_errors;

                i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->link_xon_rx, &nsd->link_xon_rx);
                i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->link_xon_tx, &nsd->link_xon_tx);
                i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
                i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->link_xoff_tx, &nsd->link_xoff_tx);

                for (i = 0; i < 8; i++) {
                        i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
                                           pf->stat_offsets_loaded,
                                           &osd->priority_xon_rx[i],
                                           &nsd->priority_xon_rx[i]);
                        i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
                                           pf->stat_offsets_loaded,
                                           &osd->priority_xon_tx[i],
                                           &nsd->priority_xon_tx[i]);
                        i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
                                           pf->stat_offsets_loaded,
                                           &osd->priority_xoff_tx[i],
                                           &nsd->priority_xoff_tx[i]);
                        i40e_stat_update32(hw,
                                           I40E_GLPRT_RXON2OFFCNT(hw->port, i),
                                           pf->stat_offsets_loaded,
                                           &osd->priority_xon_2_xoff[i],
                                           &nsd->priority_xon_2_xoff[i]);
                }

                i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
                                   I40E_GLPRT_PRC64L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_64, &nsd->rx_size_64);
                i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
                                   I40E_GLPRT_PRC127L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_127, &nsd->rx_size_127);
                i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
                                   I40E_GLPRT_PRC255L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_255, &nsd->rx_size_255);
                i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
                                   I40E_GLPRT_PRC511L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_511, &nsd->rx_size_511);
                i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
                                   I40E_GLPRT_PRC1023L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_1023, &nsd->rx_size_1023);
                i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
                                   I40E_GLPRT_PRC1522L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_1522, &nsd->rx_size_1522);
                i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
                                   I40E_GLPRT_PRC9522L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_size_big, &nsd->rx_size_big);

                i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
                                   I40E_GLPRT_PTC64L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_64, &nsd->tx_size_64);
                i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
                                   I40E_GLPRT_PTC127L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_127, &nsd->tx_size_127);
                i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
                                   I40E_GLPRT_PTC255L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_255, &nsd->tx_size_255);
                i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
                                   I40E_GLPRT_PTC511L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_511, &nsd->tx_size_511);
                i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
                                   I40E_GLPRT_PTC1023L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_1023, &nsd->tx_size_1023);
                i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
                                   I40E_GLPRT_PTC1522L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_1522, &nsd->tx_size_1522);
                i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
                                   I40E_GLPRT_PTC9522L(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->tx_size_big, &nsd->tx_size_big);

                i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_undersize, &nsd->rx_undersize);
                i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_fragments, &nsd->rx_fragments);
                i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_oversize, &nsd->rx_oversize);
                i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
                                   pf->stat_offsets_loaded,
                                   &osd->rx_jabber, &nsd->rx_jabber);
        }

        pf->stat_offsets_loaded = true;
}

/**
 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
 * @vsi: the VSI to be searched
 * @macaddr: the MAC address
 * @vlan: the vlan
 * @is_vf: make sure its a vf filter, else doesn't matter
 * @is_netdev: make sure its a netdev filter, else doesn't matter
 *
 * Returns ptr to the filter object or NULL
 **/
static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
                                                u8 *macaddr, s16 vlan,
                                                bool is_vf, bool is_netdev)
{
        struct i40e_mac_filter *f;

        if (!vsi || !macaddr)
                return NULL;

        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                if ((ether_addr_equal(macaddr, f->macaddr)) &&
                    (vlan == f->vlan)    &&
                    (!is_vf || f->is_vf) &&
                    (!is_netdev || f->is_netdev))
                        return f;
        }
        return NULL;
}

/**
 * i40e_find_mac - Find a mac addr in the macvlan filters list

 * @vsi: the VSI to be searched
 * @macaddr: the MAC address we are searching for
 * @is_vf: make sure its a vf filter, else doesn't matter
 * @is_netdev: make sure its a netdev filter, else doesn't matter
 *
 * Returns the first filter with the provided MAC address or NULL if
 * MAC address was not found
 **/
struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
                                      bool is_vf, bool is_netdev)
{
        struct i40e_mac_filter *f;

        if (!vsi || !macaddr)
                return NULL;

        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                if ((ether_addr_equal(macaddr, f->macaddr)) &&
                    (!is_vf || f->is_vf) &&
                    (!is_netdev || f->is_netdev))
                        return f;
        }
        return NULL;
}

/**
 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
 * @vsi: the VSI to be searched
 *
 * Returns true if VSI is in vlan mode or false otherwise
 **/
bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
{
        struct i40e_mac_filter *f;

        /* Only -1 for all the filters denotes not in vlan mode
         * so we have to go through all the list in order to make sure
         */
        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                if (f->vlan >= 0)
                        return true;
        }

        return false;
}

/**
 * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
 * @vsi: the VSI to be searched
 * @macaddr: the mac address to be filtered
 * @is_vf: true if it is a vf
 * @is_netdev: true if it is a netdev
 *
 * Goes through all the macvlan filters and adds a
 * macvlan filter for each unique vlan that already exists
 *
 * Returns first filter found on success, else NULL
 **/
struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
                                             bool is_vf, bool is_netdev)
{
        struct i40e_mac_filter *f;

        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                if (!i40e_find_filter(vsi, macaddr, f->vlan,
                                      is_vf, is_netdev)) {
                        if (!i40e_add_filter(vsi, macaddr, f->vlan,
                                                is_vf, is_netdev))
                                return NULL;
                }
        }

        return list_first_entry_or_null(&vsi->mac_filter_list,
                                        struct i40e_mac_filter, list);
}

/**
 * i40e_add_filter - Add a mac/vlan filter to the VSI
 * @vsi: the VSI to be searched
 * @macaddr: the MAC address
 * @vlan: the vlan
 * @is_vf: make sure its a vf filter, else doesn't matter
 * @is_netdev: make sure its a netdev filter, else doesn't matter
 *
 * Returns ptr to the filter object or NULL when no memory available.
 **/
struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
                                        u8 *macaddr, s16 vlan,
                                        bool is_vf, bool is_netdev)
{
        struct i40e_mac_filter *f;

        if (!vsi || !macaddr)
                return NULL;

        f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
        if (!f) {
                f = kzalloc(sizeof(*f), GFP_ATOMIC);
                if (!f)
                        goto add_filter_out;

                memcpy(f->macaddr, macaddr, ETH_ALEN);
                f->vlan = vlan;
                f->changed = true;

                INIT_LIST_HEAD(&f->list);
                list_add(&f->list, &vsi->mac_filter_list);
        }

        /* increment counter and add a new flag if needed */
        if (is_vf) {
                if (!f->is_vf) {
                        f->is_vf = true;
                        f->counter++;
                }
        } else if (is_netdev) {
                if (!f->is_netdev) {
                        f->is_netdev = true;
                        f->counter++;
                }
        } else {
                f->counter++;
        }

        /* changed tells sync_filters_subtask to
         * push the filter down to the firmware
         */
        if (f->changed) {
                vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
                vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
        }

add_filter_out:
        return f;
}

/**
 * i40e_del_filter - Remove a mac/vlan filter from the VSI
 * @vsi: the VSI to be searched
 * @macaddr: the MAC address
 * @vlan: the vlan
 * @is_vf: make sure it's a vf filter, else doesn't matter
 * @is_netdev: make sure it's a netdev filter, else doesn't matter
 **/
void i40e_del_filter(struct i40e_vsi *vsi,
                     u8 *macaddr, s16 vlan,
                     bool is_vf, bool is_netdev)
{
        struct i40e_mac_filter *f;

        if (!vsi || !macaddr)
                return;

        f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
        if (!f || f->counter == 0)
                return;

        if (is_vf) {
                if (f->is_vf) {
                        f->is_vf = false;
                        f->counter--;
                }
        } else if (is_netdev) {
                if (f->is_netdev) {
                        f->is_netdev = false;
                        f->counter--;
                }
        } else {
                /* make sure we don't remove a filter in use by vf or netdev */
                int min_f = 0;
                min_f += (f->is_vf ? 1 : 0);
                min_f += (f->is_netdev ? 1 : 0);

                if (f->counter > min_f)
                        f->counter--;
        }

        /* counter == 0 tells sync_filters_subtask to
         * remove the filter from the firmware's list
         */
        if (f->counter == 0) {
                f->changed = true;
                vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
                vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
        }
}

/**
 * i40e_set_mac - NDO callback to set mac address
 * @netdev: network interface device structure
 * @p: pointer to an address structure
 *
 * Returns 0 on success, negative on failure
 **/
static int i40e_set_mac(struct net_device *netdev, void *p)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;
        struct sockaddr *addr = p;
        struct i40e_mac_filter *f;

        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;

        netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);

        if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
                return 0;

        if (vsi->type == I40E_VSI_MAIN) {
                i40e_status ret;
                ret = i40e_aq_mac_address_write(&vsi->back->hw,
                                                I40E_AQC_WRITE_TYPE_LAA_ONLY,
                                                addr->sa_data, NULL);
                if (ret) {
                        netdev_info(netdev,
                                    "Addr change for Main VSI failed: %d\n",
                                    ret);
                        return -EADDRNOTAVAIL;
                }

                memcpy(vsi->back->hw.mac.addr, addr->sa_data, netdev->addr_len);
        }

        /* In order to be sure to not drop any packets, add the new address
         * then delete the old one.
         */
        f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false);
        if (!f)
                return -ENOMEM;

        i40e_sync_vsi_filters(vsi);
        i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false);
        i40e_sync_vsi_filters(vsi);

        memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);

        return 0;
}

/**
 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
 * @vsi: the VSI being setup
 * @ctxt: VSI context structure
 * @enabled_tc: Enabled TCs bitmap
 * @is_add: True if called before Add VSI
 *
 * Setup VSI queue mapping for enabled traffic classes.
 **/
static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
                                     struct i40e_vsi_context *ctxt,
                                     u8 enabled_tc,
                                     bool is_add)
{
        struct i40e_pf *pf = vsi->back;
        u16 sections = 0;
        u8 netdev_tc = 0;
        u16 numtc = 0;
        u16 qcount;
        u8 offset;
        u16 qmap;
        int i;

        sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
        offset = 0;

        if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
                /* Find numtc from enabled TC bitmap */
                for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
                        if (enabled_tc & (1 << i)) /* TC is enabled */
                                numtc++;
                }
                if (!numtc) {
                        dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
                        numtc = 1;
                }
        } else {
                /* At least TC0 is enabled in case of non-DCB case */
                numtc = 1;
        }

        vsi->tc_config.numtc = numtc;
        vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;

        /* Setup queue offset/count for all TCs for given VSI */
        for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
                /* See if the given TC is enabled for the given VSI */
                if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
                        int pow, num_qps;

                        vsi->tc_config.tc_info[i].qoffset = offset;
                        switch (vsi->type) {
                        case I40E_VSI_MAIN:
                                if (i == 0)
                                        qcount = pf->rss_size;
                                else
                                        qcount = pf->num_tc_qps;
                                vsi->tc_config.tc_info[i].qcount = qcount;
                                break;
                        case I40E_VSI_FDIR:
                        case I40E_VSI_SRIOV:
                        case I40E_VSI_VMDQ2:
                        default:
                                qcount = vsi->alloc_queue_pairs;
                                vsi->tc_config.tc_info[i].qcount = qcount;
                                WARN_ON(i != 0);
                                break;
                        }

                        /* find the power-of-2 of the number of queue pairs */
                        num_qps = vsi->tc_config.tc_info[i].qcount;
                        pow = 0;
                        while (num_qps &&
                              ((1 << pow) < vsi->tc_config.tc_info[i].qcount)) {
                                pow++;
                                num_qps >>= 1;
                        }

                        vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
                        qmap =
                            (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
                            (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);

                        offset += vsi->tc_config.tc_info[i].qcount;
                } else {
                        /* TC is not enabled so set the offset to
                         * default queue and allocate one queue
                         * for the given TC.
                         */
                        vsi->tc_config.tc_info[i].qoffset = 0;
                        vsi->tc_config.tc_info[i].qcount = 1;
                        vsi->tc_config.tc_info[i].netdev_tc = 0;

                        qmap = 0;
                }
                ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
        }

        /* Set actual Tx/Rx queue pairs */
        vsi->num_queue_pairs = offset;

        /* Scheduler section valid can only be set for ADD VSI */
        if (is_add) {
                sections |= I40E_AQ_VSI_PROP_SCHED_VALID;

                ctxt->info.up_enable_bits = enabled_tc;
        }
        if (vsi->type == I40E_VSI_SRIOV) {
                ctxt->info.mapping_flags |=
                                     cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
                for (i = 0; i < vsi->num_queue_pairs; i++)
                        ctxt->info.queue_mapping[i] =
                                               cpu_to_le16(vsi->base_queue + i);
        } else {
                ctxt->info.mapping_flags |=
                                        cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
                ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
        }
        ctxt->info.valid_sections |= cpu_to_le16(sections);
}

/**
 * i40e_set_rx_mode - NDO callback to set the netdev filters
 * @netdev: network interface device structure
 **/
static void i40e_set_rx_mode(struct net_device *netdev)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_mac_filter *f, *ftmp;
        struct i40e_vsi *vsi = np->vsi;
        struct netdev_hw_addr *uca;
        struct netdev_hw_addr *mca;
        struct netdev_hw_addr *ha;

        /* add addr if not already in the filter list */
        netdev_for_each_uc_addr(uca, netdev) {
                if (!i40e_find_mac(vsi, uca->addr, false, true)) {
                        if (i40e_is_vsi_in_vlan(vsi))
                                i40e_put_mac_in_vlan(vsi, uca->addr,
                                                     false, true);
                        else
                                i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
                                                false, true);
                }
        }

        netdev_for_each_mc_addr(mca, netdev) {
                if (!i40e_find_mac(vsi, mca->addr, false, true)) {
                        if (i40e_is_vsi_in_vlan(vsi))
                                i40e_put_mac_in_vlan(vsi, mca->addr,
                                                     false, true);
                        else
                                i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
                                                false, true);
                }
        }

        /* remove filter if not in netdev list */
        list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
                bool found = false;

                if (!f->is_netdev)
                        continue;

                if (is_multicast_ether_addr(f->macaddr)) {
                        netdev_for_each_mc_addr(mca, netdev) {
                                if (ether_addr_equal(mca->addr, f->macaddr)) {
                                        found = true;
                                        break;
                                }
                        }
                } else {
                        netdev_for_each_uc_addr(uca, netdev) {
                                if (ether_addr_equal(uca->addr, f->macaddr)) {
                                        found = true;
                                        break;
                                }
                        }

                        for_each_dev_addr(netdev, ha) {
                                if (ether_addr_equal(ha->addr, f->macaddr)) {
                                        found = true;
                                        break;
                                }
                        }
                }
                if (!found)
                        i40e_del_filter(
                           vsi, f->macaddr, I40E_VLAN_ANY, false, true);
        }

        /* check for other flag changes */
        if (vsi->current_netdev_flags != vsi->netdev->flags) {
                vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
                vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
        }
}

/**
 * i40e_sync_vsi_filters - Update the VSI filter list to the HW
 * @vsi: ptr to the VSI
 *
 * Push any outstanding VSI filter changes through the AdminQ.
 *
 * Returns 0 or error value
 **/
int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
{
        struct i40e_mac_filter *f, *ftmp;
        bool promisc_forced_on = false;
        bool add_happened = false;
        int filter_list_len = 0;
        u32 changed_flags = 0;
        i40e_status aq_ret = 0;
        struct i40e_pf *pf;
        int num_add = 0;
        int num_del = 0;
        u16 cmd_flags;

        /* empty array typed pointers, kcalloc later */
        struct i40e_aqc_add_macvlan_element_data *add_list;
        struct i40e_aqc_remove_macvlan_element_data *del_list;

        while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
                usleep_range(1000, 2000);
        pf = vsi->back;

        if (vsi->netdev) {
                changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
                vsi->current_netdev_flags = vsi->netdev->flags;
        }

        if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
                vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;

                filter_list_len = pf->hw.aq.asq_buf_size /
                            sizeof(struct i40e_aqc_remove_macvlan_element_data);
                del_list = kcalloc(filter_list_len,
                            sizeof(struct i40e_aqc_remove_macvlan_element_data),
                            GFP_KERNEL);
                if (!del_list)
                        return -ENOMEM;

                list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
                        if (!f->changed)
                                continue;

                        if (f->counter != 0)
                                continue;
                        f->changed = false;
                        cmd_flags = 0;

                        /* add to delete list */
                        memcpy(del_list[num_del].mac_addr,
                               f->macaddr, ETH_ALEN);
                        del_list[num_del].vlan_tag =
                                cpu_to_le16((u16)(f->vlan ==
                                            I40E_VLAN_ANY ? 0 : f->vlan));

                        /* vlan0 as wild card to allow packets from all vlans */
                        if (f->vlan == I40E_VLAN_ANY ||
                            (vsi->netdev && !(vsi->netdev->features &
                                              NETIF_F_HW_VLAN_CTAG_FILTER)))
                                cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
                        cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
                        del_list[num_del].flags = cmd_flags;
                        num_del++;

                        /* unlink from filter list */
                        list_del(&f->list);
                        kfree(f);

                        /* flush a full buffer */
                        if (num_del == filter_list_len) {
                                aq_ret = i40e_aq_remove_macvlan(&pf->hw,
                                            vsi->seid, del_list, num_del,
                                            NULL);
                                num_del = 0;
                                memset(del_list, 0, sizeof(*del_list));

                                if (aq_ret)
                                        dev_info(&pf->pdev->dev,
                                                 "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
                                                 aq_ret,
                                                 pf->hw.aq.asq_last_status);
                        }
                }
                if (num_del) {
                        aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
                                                     del_list, num_del, NULL);
                        num_del = 0;

                        if (aq_ret)
                                dev_info(&pf->pdev->dev,
                                         "ignoring delete macvlan error, err %d, aq_err %d\n",
                                         aq_ret, pf->hw.aq.asq_last_status);
                }

                kfree(del_list);
                del_list = NULL;

                /* do all the adds now */
                filter_list_len = pf->hw.aq.asq_buf_size /
                               sizeof(struct i40e_aqc_add_macvlan_element_data),
                add_list = kcalloc(filter_list_len,
                               sizeof(struct i40e_aqc_add_macvlan_element_data),
                               GFP_KERNEL);
                if (!add_list)
                        return -ENOMEM;

                list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
                        if (!f->changed)
                                continue;

                        if (f->counter == 0)
                                continue;
                        f->changed = false;
                        add_happened = true;
                        cmd_flags = 0;

                        /* add to add array */
                        memcpy(add_list[num_add].mac_addr,
                               f->macaddr, ETH_ALEN);
                        add_list[num_add].vlan_tag =
                                cpu_to_le16(
                                 (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
                        add_list[num_add].queue_number = 0;

                        cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;

                        /* vlan0 as wild card to allow packets from all vlans */
                        if (f->vlan == I40E_VLAN_ANY || (vsi->netdev &&
                            !(vsi->netdev->features &
                                                 NETIF_F_HW_VLAN_CTAG_FILTER)))
                                cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
                        add_list[num_add].flags = cpu_to_le16(cmd_flags);
                        num_add++;

                        /* flush a full buffer */
                        if (num_add == filter_list_len) {
                                aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
                                                             add_list, num_add,
                                                             NULL);
                                num_add = 0;

                                if (aq_ret)
                                        break;
                                memset(add_list, 0, sizeof(*add_list));
                        }
                }
                if (num_add) {
                        aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
                                                     add_list, num_add, NULL);
                        num_add = 0;
                }
                kfree(add_list);
                add_list = NULL;

                if (add_happened && (!aq_ret)) {
                        /* do nothing */;
                } else if (add_happened && (aq_ret)) {
                        dev_info(&pf->pdev->dev,
                                 "add filter failed, err %d, aq_err %d\n",
                                 aq_ret, pf->hw.aq.asq_last_status);
                        if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
                            !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
                                      &vsi->state)) {
                                promisc_forced_on = true;
                                set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
                                        &vsi->state);
                                dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
                        }
                }
        }

        /* check for changes in promiscuous modes */
        if (changed_flags & IFF_ALLMULTI) {
                bool cur_multipromisc;
                cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
                aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
                                                               vsi->seid,
                                                               cur_multipromisc,
                                                               NULL);
                if (aq_ret)
                        dev_info(&pf->pdev->dev,
                                 "set multi promisc failed, err %d, aq_err %d\n",
                                 aq_ret, pf->hw.aq.asq_last_status);
        }
        if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
                bool cur_promisc;
                cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
                               test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
                                        &vsi->state));
                aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
                                                             vsi->seid,
                                                             cur_promisc, NULL);
                if (aq_ret)
                        dev_info(&pf->pdev->dev,
                                 "set uni promisc failed, err %d, aq_err %d\n",
                                 aq_ret, pf->hw.aq.asq_last_status);
        }

        clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
        return 0;
}

/**
 * i40e_sync_filters_subtask - Sync the VSI filter list with HW
 * @pf: board private structure
 **/
static void i40e_sync_filters_subtask(struct i40e_pf *pf)
{
        int v;

        if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
                return;
        pf->flags &= ~I40E_FLAG_FILTER_SYNC;

        for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
                if (pf->vsi[v] &&
                    (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
                        i40e_sync_vsi_filters(pf->vsi[v]);
        }
}

/**
 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
 * @netdev: network interface device structure
 * @new_mtu: new value for maximum frame size
 *
 * Returns 0 on success, negative on failure
 **/
static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
        struct i40e_vsi *vsi = np->vsi;

        /* MTU < 68 is an error and causes problems on some kernels */
        if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
                return -EINVAL;

        netdev_info(netdev, "changing MTU from %d to %d\n",
                    netdev->mtu, new_mtu);
        netdev->mtu = new_mtu;
        if (netif_running(netdev))
                i40e_vsi_reinit_locked(vsi);

        return 0;
}

/**
 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
 * @vsi: the vsi being adjusted
 **/
void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
{
        struct i40e_vsi_context ctxt;
        i40e_status ret;

        if ((vsi->info.valid_sections &
             cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
            ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
                return;  /* already enabled */

        vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
        vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
                                    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;

        ctxt.seid = vsi->seid;
        memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
        ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
        if (ret) {
                dev_info(&vsi->back->pdev->dev,
                         "%s: update vsi failed, aq_err=%d\n",
                         __func__, vsi->back->hw.aq.asq_last_status);
        }
}

/**
 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
 * @vsi: the vsi being adjusted
 **/
void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
{
        struct i40e_vsi_context ctxt;
        i40e_status ret;

        if ((vsi->info.valid_sections &
             cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
            ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
             I40E_AQ_VSI_PVLAN_EMOD_MASK))
                return;  /* already disabled */

        vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
        vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
                                    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;

        ctxt.seid = vsi->seid;
        memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
        ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
        if (ret) {
                dev_info(&vsi->back->pdev->dev,
                         "%s: update vsi failed, aq_err=%d\n",
                         __func__, vsi->back->hw.aq.asq_last_status);
        }
}

/**
 * i40e_vlan_rx_register - Setup or shutdown vlan offload
 * @netdev: network interface to be adjusted
 * @features: netdev features to test if VLAN offload is enabled or not
 **/
static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;

        if (features & NETIF_F_HW_VLAN_CTAG_RX)
                i40e_vlan_stripping_enable(vsi);
        else
                i40e_vlan_stripping_disable(vsi);
}

/**
 * i40e_vsi_add_vlan - Add vsi membership for given vlan
 * @vsi: the vsi being configured
 * @vid: vlan id to be added (0 = untagged only , -1 = any)
 **/
int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
{
        struct i40e_mac_filter *f, *add_f;
        bool is_netdev, is_vf;
        int ret;

        is_vf = (vsi->type == I40E_VSI_SRIOV);
        is_netdev = !!(vsi->netdev);

        if (is_netdev) {
                add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
                                        is_vf, is_netdev);
                if (!add_f) {
                        dev_info(&vsi->back->pdev->dev,
                                 "Could not add vlan filter %d for %pM\n",
                                 vid, vsi->netdev->dev_addr);
                        return -ENOMEM;
                }
        }

        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
                if (!add_f) {
                        dev_info(&vsi->back->pdev->dev,
                                 "Could not add vlan filter %d for %pM\n",
                                 vid, f->macaddr);
                        return -ENOMEM;
                }
        }

        ret = i40e_sync_vsi_filters(vsi);
        if (ret) {
                dev_info(&vsi->back->pdev->dev,
                         "Could not sync filters for vid %d\n", vid);
                return ret;
        }

        /* Now if we add a vlan tag, make sure to check if it is the first
         * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
         * with 0, so we now accept untagged and specified tagged traffic
         * (and not any taged and untagged)
         */
        if (vid > 0) {
                if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
                                                  I40E_VLAN_ANY,
                                                  is_vf, is_netdev)) {
                        i40e_del_filter(vsi, vsi->netdev->dev_addr,
                                        I40E_VLAN_ANY, is_vf, is_netdev);
                        add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
                                                is_vf, is_netdev);
                        if (!add_f) {
                                dev_info(&vsi->back->pdev->dev,
                                         "Could not add filter 0 for %pM\n",
                                         vsi->netdev->dev_addr);
                                return -ENOMEM;
                        }
                }

                list_for_each_entry(f, &vsi->mac_filter_list, list) {
                        if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
                                             is_vf, is_netdev)) {
                                i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
                                                is_vf, is_netdev);
                                add_f = i40e_add_filter(vsi, f->macaddr,
                                                        0, is_vf, is_netdev);
                                if (!add_f) {
                                        dev_info(&vsi->back->pdev->dev,
                                                 "Could not add filter 0 for %pM\n",
                                                 f->macaddr);
                                        return -ENOMEM;
                                }
                        }
                }
                ret = i40e_sync_vsi_filters(vsi);
        }

        return ret;
}

/**
 * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
 * @vsi: the vsi being configured
 * @vid: vlan id to be removed (0 = untagged only , -1 = any)
 *
 * Return: 0 on success or negative otherwise
 **/
int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
{
        struct net_device *netdev = vsi->netdev;
        struct i40e_mac_filter *f, *add_f;
        bool is_vf, is_netdev;
        int filter_count = 0;
        int ret;

        is_vf = (vsi->type == I40E_VSI_SRIOV);
        is_netdev = !!(netdev);

        if (is_netdev)
                i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);

        list_for_each_entry(f, &vsi->mac_filter_list, list)
                i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);

        ret = i40e_sync_vsi_filters(vsi);
        if (ret) {
                dev_info(&vsi->back->pdev->dev, "Could not sync filters\n");
                return ret;
        }

        /* go through all the filters for this VSI and if there is only
         * vid == 0 it means there are no other filters, so vid 0 must
         * be replaced with -1. This signifies that we should from now
         * on accept any traffic (with any tag present, or untagged)
         */
        list_for_each_entry(f, &vsi->mac_filter_list, list) {
                if (is_netdev) {
                        if (f->vlan &&
                            ether_addr_equal(netdev->dev_addr, f->macaddr))
                                filter_count++;
                }

                if (f->vlan)
                        filter_count++;
        }

        if (!filter_count && is_netdev) {
                i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
                f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
                                    is_vf, is_netdev);
                if (!f) {
                        dev_info(&vsi->back->pdev->dev,
                                 "Could not add filter %d for %pM\n",
                                 I40E_VLAN_ANY, netdev->dev_addr);
                        return -ENOMEM;
                }
        }

        if (!filter_count) {
                list_for_each_entry(f, &vsi->mac_filter_list, list) {
                        i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
                        add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
                                            is_vf, is_netdev);
                        if (!add_f) {
                                dev_info(&vsi->back->pdev->dev,
                                         "Could not add filter %d for %pM\n",
                                         I40E_VLAN_ANY, f->macaddr);
                                return -ENOMEM;
                        }
                }
        }

        return i40e_sync_vsi_filters(vsi);
}

/**
 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
 * @netdev: network interface to be adjusted
 * @vid: vlan id to be added
 *
 * net_device_ops implementation for adding vlan ids
 **/
static int i40e_vlan_rx_add_vid(struct net_device *netdev,
                                __always_unused __be16 proto, u16 vid)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;
        int ret = 0;

        if (vid > 4095)
                return -EINVAL;

        netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);

        /* If the network stack called us with vid = 0, we should
         * indicate to i40e_vsi_add_vlan() that we want to receive
         * any traffic (i.e. with any vlan tag, or untagged)
         */
        ret = i40e_vsi_add_vlan(vsi, vid ? vid : I40E_VLAN_ANY);

        if (!ret && (vid < VLAN_N_VID))
                set_bit(vid, vsi->active_vlans);

        return ret;
}

/**
 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
 * @netdev: network interface to be adjusted
 * @vid: vlan id to be removed
 *
 * net_device_ops implementation for adding vlan ids
 **/
static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
                                 __always_unused __be16 proto, u16 vid)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;

        netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);

        /* return code is ignored as there is nothing a user
         * can do about failure to remove and a log message was
         * already printed from the other function
         */
        i40e_vsi_kill_vlan(vsi, vid);

        clear_bit(vid, vsi->active_vlans);

        return 0;
}

/**
 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
 * @vsi: the vsi being brought back up
 **/
static void i40e_restore_vlan(struct i40e_vsi *vsi)
{
        u16 vid;

        if (!vsi->netdev)
                return;

        i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);

        for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
                i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
                                     vid);
}

/**
 * i40e_vsi_add_pvid - Add pvid for the VSI

 * @vsi: the vsi being adjusted
 * @vid: the vlan id to set as a PVID
 **/
int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
{
        struct i40e_vsi_context ctxt;
        i40e_status aq_ret;

        vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
        vsi->info.pvid = cpu_to_le16(vid);
        vsi->info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID;
        vsi->info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;

        ctxt.seid = vsi->seid;
        memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
        aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
        if (aq_ret) {
                dev_info(&vsi->back->pdev->dev,
                         "%s: update vsi failed, aq_err=%d\n",
                         __func__, vsi->back->hw.aq.asq_last_status);
                return -ENOENT;
        }

        return 0;
}

/**
 * i40e_vsi_remove_pvid - Remove the pvid from the VSI
 * @vsi: the vsi being adjusted
 *
 * Just use the vlan_rx_register() service to put it back to normal
 **/
void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
{
        vsi->info.pvid = 0;
        i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
}

/**
 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
 * @vsi: ptr to the VSI
 *
 * If this function returns with an error, then it's possible one or
 * more of the rings is populated (while the rest are not).  It is the
 * callers duty to clean those orphaned rings.
 *
 * Return 0 on success, negative on failure
 **/
static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
{
        int i, err = 0;

        for (i = 0; i < vsi->num_queue_pairs && !err; i++)
                err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);

        return err;
}

/**
 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
 * @vsi: ptr to the VSI
 *
 * Free VSI's transmit software resources
 **/
static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
{
        int i;

        for (i = 0; i < vsi->num_queue_pairs; i++)
                if (vsi->tx_rings[i]->desc)
                        i40e_free_tx_resources(vsi->tx_rings[i]);
}

/**
 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
 * @vsi: ptr to the VSI
 *
 * If this function returns with an error, then it's possible one or
 * more of the rings is populated (while the rest are not).  It is the
 * callers duty to clean those orphaned rings.
 *
 * Return 0 on success, negative on failure
 **/
static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
{
        int i, err = 0;

        for (i = 0; i < vsi->num_queue_pairs && !err; i++)
                err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
        return err;
}

/**
 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
 * @vsi: ptr to the VSI
 *
 * Free all receive software resources
 **/
static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
{
        int i;

        for (i = 0; i < vsi->num_queue_pairs; i++)
                if (vsi->rx_rings[i]->desc)
                        i40e_free_rx_resources(vsi->rx_rings[i]);
}

/**
 * i40e_configure_tx_ring - Configure a transmit ring context and rest
 * @ring: The Tx ring to configure
 *
 * Configure the Tx descriptor ring in the HMC context.
 **/
static int i40e_configure_tx_ring(struct i40e_ring *ring)
{
        struct i40e_vsi *vsi = ring->vsi;
        u16 pf_q = vsi->base_queue + ring->queue_index;
        struct i40e_hw *hw = &vsi->back->hw;
        struct i40e_hmc_obj_txq tx_ctx;
        i40e_status err = 0;
        u32 qtx_ctl = 0;

        /* some ATR related tx ring init */
        if (vsi->back->flags & I40E_FLAG_FDIR_ATR_ENABLED) {
                ring->atr_sample_rate = vsi->back->atr_sample_rate;
                ring->atr_count = 0;
        } else {
                ring->atr_sample_rate = 0;
        }

        /* initialize XPS */
        if (ring->q_vector && ring->netdev &&
            !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
                netif_set_xps_queue(ring->netdev,
                                    &ring->q_vector->affinity_mask,
                                    ring->queue_index);

        /* clear the context structure first */
        memset(&tx_ctx, 0, sizeof(tx_ctx));

        tx_ctx.new_context = 1;
        tx_ctx.base = (ring->dma / 128);
        tx_ctx.qlen = ring->count;
        tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FDIR_ENABLED |
                        I40E_FLAG_FDIR_ATR_ENABLED));

        /* As part of VSI creation/update, FW allocates certain
         * Tx arbitration queue sets for each TC enabled for
         * the VSI. The FW returns the handles to these queue
         * sets as part of the response buffer to Add VSI,
         * Update VSI, etc. AQ commands. It is expected that
         * these queue set handles be associated with the Tx
         * queues by the driver as part of the TX queue context
         * initialization. This has to be done regardless of
         * DCB as by default everything is mapped to TC0.
         */
        tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
        tx_ctx.rdylist_act = 0;

        /* clear the context in the HMC */
        err = i40e_clear_lan_tx_queue_context(hw, pf_q);
        if (err) {
                dev_info(&vsi->back->pdev->dev,
                         "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
                         ring->queue_index, pf_q, err);
                return -ENOMEM;
        }

        /* set the context in the HMC */
        err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
        if (err) {
                dev_info(&vsi->back->pdev->dev,
                         "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
                         ring->queue_index, pf_q, err);
                return -ENOMEM;
        }

        /* Now associate this queue with this PCI function */
        qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
        qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
                    I40E_QTX_CTL_PF_INDX_MASK);
        wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
        i40e_flush(hw);

        clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);

        /* cache tail off for easier writes later */
        ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);

        return 0;
}

/**
 * i40e_configure_rx_ring - Configure a receive ring context
 * @ring: The Rx ring to configure
 *
 * Configure the Rx descriptor ring in the HMC context.
 **/
static int i40e_configure_rx_ring(struct i40e_ring *ring)
{
        struct i40e_vsi *vsi = ring->vsi;
        u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
        u16 pf_q = vsi->base_queue + ring->queue_index;
        struct i40e_hw *hw = &vsi->back->hw;
        struct i40e_hmc_obj_rxq rx_ctx;
        i40e_status err = 0;

        ring->state = 0;

        /* clear the context structure first */
        memset(&rx_ctx, 0, sizeof(rx_ctx));

        ring->rx_buf_len = vsi->rx_buf_len;
        ring->rx_hdr_len = vsi->rx_hdr_len;

        rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
        rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;

        rx_ctx.base = (ring->dma / 128);
        rx_ctx.qlen = ring->count;

        if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
                set_ring_16byte_desc_enabled(ring);
                rx_ctx.dsize = 0;
        } else {
                rx_ctx.dsize = 1;
        }

        rx_ctx.dtype = vsi->dtype;
        if (vsi->dtype) {
                set_ring_ps_enabled(ring);
                rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
                                  I40E_RX_SPLIT_IP      |
                                  I40E_RX_SPLIT_TCP_UDP |
                                  I40E_RX_SPLIT_SCTP;
        } else {
                rx_ctx.hsplit_0 = 0;
        }

        rx_ctx.rxmax = min_t(u16, vsi->max_frame,
                                  (chain_len * ring->rx_buf_len));
        rx_ctx.tphrdesc_ena = 1;
        rx_ctx.tphwdesc_ena = 1;
        rx_ctx.tphdata_ena = 1;
        rx_ctx.tphhead_ena = 1;
        rx_ctx.lrxqthresh = 2;
        rx_ctx.crcstrip = 1;
        rx_ctx.l2tsel = 1;
        rx_ctx.showiv = 1;

        /* clear the context in the HMC */
        err = i40e_clear_lan_rx_queue_context(hw, pf_q);
        if (err) {
                dev_info(&vsi->back->pdev->dev,
                         "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
                         ring->queue_index, pf_q, err);
                return -ENOMEM;
        }

        /* set the context in the HMC */
        err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
        if (err) {
                dev_info(&vsi->back->pdev->dev,
                         "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
                         ring->queue_index, pf_q, err);
                return -ENOMEM;
        }

        /* cache tail for quicker writes, and clear the reg before use */
        ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
        writel(0, ring->tail);

        i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));

        return 0;
}

/**
 * i40e_vsi_configure_tx - Configure the VSI for Tx
 * @vsi: VSI structure describing this set of rings and resources
 *
 * Configure the Tx VSI for operation.
 **/
static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
{
        int err = 0;
        u16 i;

        for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
                err = i40e_configure_tx_ring(vsi->tx_rings[i]);

        return err;
}

/**
 * i40e_vsi_configure_rx - Configure the VSI for Rx
 * @vsi: the VSI being configured
 *
 * Configure the Rx VSI for operation.
 **/
static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
{
        int err = 0;
        u16 i;

        if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
                vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
                               + ETH_FCS_LEN + VLAN_HLEN;
        else
                vsi->max_frame = I40E_RXBUFFER_2048;

        /* figure out correct receive buffer length */
        switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
                                    I40E_FLAG_RX_PS_ENABLED)) {
        case I40E_FLAG_RX_1BUF_ENABLED:
                vsi->rx_hdr_len = 0;
                vsi->rx_buf_len = vsi->max_frame;
                vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
                break;
        case I40E_FLAG_RX_PS_ENABLED:
                vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
                vsi->rx_buf_len = I40E_RXBUFFER_2048;
                vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
                break;
        default:
                vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
                vsi->rx_buf_len = I40E_RXBUFFER_2048;
                vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
                break;
        }

        /* round up for the chip's needs */
        vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
                                (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
        vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
                                (1 << I40E_RXQ_CTX_DBUFF_SHIFT));

        /* set up individual rings */
        for (i = 0; i < vsi->num_queue_pairs && !err; i++)
                err = i40e_configure_rx_ring(vsi->rx_rings[i]);

        return err;
}

/**
 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
 * @vsi: ptr to the VSI
 **/
static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
{
        u16 qoffset, qcount;
        int i, n;

        if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
                return;

        for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
                if (!(vsi->tc_config.enabled_tc & (1 << n)))
                        continue;

                qoffset = vsi->tc_config.tc_info[n].qoffset;
                qcount = vsi->tc_config.tc_info[n].qcount;
                for (i = qoffset; i < (qoffset + qcount); i++) {
                        struct i40e_ring *rx_ring = vsi->rx_rings[i];
                        struct i40e_ring *tx_ring = vsi->tx_rings[i];
                        rx_ring->dcb_tc = n;
                        tx_ring->dcb_tc = n;
                }
        }
}

/**
 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
 * @vsi: ptr to the VSI
 **/
static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
{
        if (vsi->netdev)
                i40e_set_rx_mode(vsi->netdev);
}

/**
 * i40e_vsi_configure - Set up the VSI for action
 * @vsi: the VSI being configured
 **/
static int i40e_vsi_configure(struct i40e_vsi *vsi)
{
        int err;

        i40e_set_vsi_rx_mode(vsi);
        i40e_restore_vlan(vsi);
        i40e_vsi_config_dcb_rings(vsi);
        err = i40e_vsi_configure_tx(vsi);
        if (!err)
                err = i40e_vsi_configure_rx(vsi);

        return err;
}

/**
 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
 * @vsi: the VSI being configured
 **/
static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
{
        struct i40e_pf *pf = vsi->back;
        struct i40e_q_vector *q_vector;
        struct i40e_hw *hw = &pf->hw;
        u16 vector;
        int i, q;
        u32 val;
        u32 qp;

        /* The interrupt indexing is offset by 1 in the PFINT_ITRn
         * and PFINT_LNKLSTn registers, e.g.:
         *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
         */
        qp = vsi->base_queue;
        vector = vsi->base_vector;
        for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
                q_vector = vsi->q_vectors[i];
                q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
                q_vector->rx.latency_range = I40E_LOW_LATENCY;
                wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
                     q_vector->rx.itr);
                q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
                q_vector->tx.latency_range = I40E_LOW_LATENCY;
                wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
                     q_vector->tx.itr);

                /* Linked list for the queuepairs assigned to this vector */
                wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
                for (q = 0; q < q_vector->num_ringpairs; q++) {
                        val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
                              (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
                              (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
                              (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
                              (I40E_QUEUE_TYPE_TX
                                      << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);

                        wr32(hw, I40E_QINT_RQCTL(qp), val);

                        val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
                              (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
                              (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
                              ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
                              (I40E_QUEUE_TYPE_RX
                                      << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);

                        /* Terminate the linked list */
                        if (q == (q_vector->num_ringpairs - 1))
                                val |= (I40E_QUEUE_END_OF_LIST
                                           << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);

                        wr32(hw, I40E_QINT_TQCTL(qp), val);
                        qp++;
                }
        }

        i40e_flush(hw);
}

/**
 * i40e_enable_misc_int_causes - enable the non-queue interrupts
 * @hw: ptr to the hardware info
 **/
static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
{
        u32 val;

        /* clear things first */
        wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
        rd32(hw, I40E_PFINT_ICR0);         /* read to clear */

        val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
              I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
              I40E_PFINT_ICR0_ENA_GRST_MASK          |
              I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
              I40E_PFINT_ICR0_ENA_GPIO_MASK          |
              I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK  |
              I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
              I40E_PFINT_ICR0_ENA_VFLR_MASK          |
              I40E_PFINT_ICR0_ENA_ADMINQ_MASK;

        wr32(hw, I40E_PFINT_ICR0_ENA, val);

        /* SW_ITR_IDX = 0, but don't change INTENA */
        wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
                                        I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);

        /* OTHER_ITR_IDX = 0 */
        wr32(hw, I40E_PFINT_STAT_CTL0, 0);
}

/**
 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
 * @vsi: the VSI being configured
 **/
static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
{
        struct i40e_q_vector *q_vector = vsi->q_vectors[0];
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        u32 val;

        /* set the ITR configuration */
        q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
        q_vector->rx.latency_range = I40E_LOW_LATENCY;
        wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
        q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
        q_vector->tx.latency_range = I40E_LOW_LATENCY;
        wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);

        i40e_enable_misc_int_causes(hw);

        /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
        wr32(hw, I40E_PFINT_LNKLST0, 0);

        /* Associate the queue pair to the vector and enable the q int */
        val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
              (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
              (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);

        wr32(hw, I40E_QINT_RQCTL(0), val);

        val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
              (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
              (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);

        wr32(hw, I40E_QINT_TQCTL(0), val);
        i40e_flush(hw);
}

/**
 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
 * @pf: board private structure
 **/
void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
{
        struct i40e_hw *hw = &pf->hw;
        u32 val;

        val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
              I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
              (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);

        wr32(hw, I40E_PFINT_DYN_CTL0, val);
        i40e_flush(hw);
}

/**
 * i40e_irq_dynamic_enable - Enable default interrupt generation settings
 * @vsi: pointer to a vsi
 * @vector: enable a particular Hw Interrupt vector
 **/
void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
{
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        u32 val;

        val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
              I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
              (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
        wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
        /* skip the flush */
}

/**
 * i40e_msix_clean_rings - MSIX mode Interrupt Handler
 * @irq: interrupt number
 * @data: pointer to a q_vector
 **/
static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
{
        struct i40e_q_vector *q_vector = data;

        if (!q_vector->tx.ring && !q_vector->rx.ring)
                return IRQ_HANDLED;

        napi_schedule(&q_vector->napi);

        return IRQ_HANDLED;
}

/**
 * i40e_fdir_clean_rings - Interrupt Handler for FDIR rings
 * @irq: interrupt number
 * @data: pointer to a q_vector
 **/
static irqreturn_t i40e_fdir_clean_rings(int irq, void *data)
{
        struct i40e_q_vector *q_vector = data;

        if (!q_vector->tx.ring && !q_vector->rx.ring)
                return IRQ_HANDLED;

        pr_info("fdir ring cleaning needed\n");

        return IRQ_HANDLED;
}

/**
 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
 * @vsi: the VSI being configured
 * @basename: name for the vector
 *
 * Allocates MSI-X vectors and requests interrupts from the kernel.
 **/
static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
{
        int q_vectors = vsi->num_q_vectors;
        struct i40e_pf *pf = vsi->back;
        int base = vsi->base_vector;
        int rx_int_idx = 0;
        int tx_int_idx = 0;
        int vector, err;

        for (vector = 0; vector < q_vectors; vector++) {
                struct i40e_q_vector *q_vector = vsi->q_vectors[vector];

                if (q_vector->tx.ring && q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                                 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
                        tx_int_idx++;
                } else if (q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                                 "%s-%s-%d", basename, "rx", rx_int_idx++);
                } else if (q_vector->tx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name) - 1,
                                 "%s-%s-%d", basename, "tx", tx_int_idx++);
                } else {
                        /* skip this unused q_vector */
                        continue;
                }
                err = request_irq(pf->msix_entries[base + vector].vector,
                                  vsi->irq_handler,
                                  0,
                                  q_vector->name,
                                  q_vector);
                if (err) {
                        dev_info(&pf->pdev->dev,
                                 "%s: request_irq failed, error: %d\n",
                                 __func__, err);
                        goto free_queue_irqs;
                }
                /* assign the mask for this irq */
                irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
                                      &q_vector->affinity_mask);
        }

        return 0;

free_queue_irqs:
        while (vector) {
                vector--;
                irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
                                      NULL);
                free_irq(pf->msix_entries[base + vector].vector,
                         &(vsi->q_vectors[vector]));
        }
        return err;
}

/**
 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
 * @vsi: the VSI being un-configured
 **/
static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
{
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        int base = vsi->base_vector;
        int i;

        for (i = 0; i < vsi->num_queue_pairs; i++) {
                wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
                wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
        }

        if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
                for (i = vsi->base_vector;
                     i < (vsi->num_q_vectors + vsi->base_vector); i++)
                        wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);

                i40e_flush(hw);
                for (i = 0; i < vsi->num_q_vectors; i++)
                        synchronize_irq(pf->msix_entries[i + base].vector);
        } else {
                /* Legacy and MSI mode - this stops all interrupt handling */
                wr32(hw, I40E_PFINT_ICR0_ENA, 0);
                wr32(hw, I40E_PFINT_DYN_CTL0, 0);
                i40e_flush(hw);
                synchronize_irq(pf->pdev->irq);
        }
}

/**
 * i40e_vsi_enable_irq - Enable IRQ for the given VSI
 * @vsi: the VSI being configured
 **/
static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
{
        struct i40e_pf *pf = vsi->back;
        int i;

        if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
                for (i = vsi->base_vector;
                     i < (vsi->num_q_vectors + vsi->base_vector); i++)
                        i40e_irq_dynamic_enable(vsi, i);
        } else {
                i40e_irq_dynamic_enable_icr0(pf);
        }

        i40e_flush(&pf->hw);
        return 0;
}

/**
 * i40e_stop_misc_vector - Stop the vector that handles non-queue events
 * @pf: board private structure
 **/
static void i40e_stop_misc_vector(struct i40e_pf *pf)
{
        /* Disable ICR 0 */
        wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
        i40e_flush(&pf->hw);
}

/**
 * i40e_intr - MSI/Legacy and non-queue interrupt handler
 * @irq: interrupt number
 * @data: pointer to a q_vector
 *
 * This is the handler used for all MSI/Legacy interrupts, and deals
 * with both queue and non-queue interrupts.  This is also used in
 * MSIX mode to handle the non-queue interrupts.
 **/
static irqreturn_t i40e_intr(int irq, void *data)
{
        struct i40e_pf *pf = (struct i40e_pf *)data;
        struct i40e_hw *hw = &pf->hw;
        u32 icr0, icr0_remaining;
        u32 val, ena_mask;

        icr0 = rd32(hw, I40E_PFINT_ICR0);

        val = rd32(hw, I40E_PFINT_DYN_CTL0);
        val = val | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
        wr32(hw, I40E_PFINT_DYN_CTL0, val);

        /* if sharing a legacy IRQ, we might get called w/o an intr pending */
        if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
                return IRQ_NONE;

        ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);

        /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
        if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {

                /* temporarily disable queue cause for NAPI processing */
                u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
                qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
                wr32(hw, I40E_QINT_RQCTL(0), qval);

                qval = rd32(hw, I40E_QINT_TQCTL(0));
                qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
                wr32(hw, I40E_QINT_TQCTL(0), qval);

                if (!test_bit(__I40E_DOWN, &pf->state))
                        napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
        }

        if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
                ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
                set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
        }

        if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
                ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
                set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
        }

        if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
                ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
                set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
        }

        if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
                if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
                        set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
                ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
                val = rd32(hw, I40E_GLGEN_RSTAT);
                val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
                       >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
                if (val & I40E_RESET_CORER)
                        pf->corer_count++;
                else if (val & I40E_RESET_GLOBR)
                        pf->globr_count++;
                else if (val & I40E_RESET_EMPR)
                        pf->empr_count++;
        }

        /* If a critical error is pending we have no choice but to reset the
         * device.
         * Report and mask out any remaining unexpected interrupts.
         */
        icr0_remaining = icr0 & ena_mask;
        if (icr0_remaining) {
                dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
                         icr0_remaining);
                if ((icr0_remaining & I40E_PFINT_ICR0_HMC_ERR_MASK) ||
                    (icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
                    (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
                    (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK) ||
                    (icr0_remaining & I40E_PFINT_ICR0_MAL_DETECT_MASK)) {
                        if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
                                dev_info(&pf->pdev->dev, "HMC error interrupt\n");
                        } else {
                                dev_info(&pf->pdev->dev, "device will be reset\n");
                                set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
                                i40e_service_event_schedule(pf);
                        }
                }
                ena_mask &= ~icr0_remaining;
        }

        /* re-enable interrupt causes */
        wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
        if (!test_bit(__I40E_DOWN, &pf->state)) {
                i40e_service_event_schedule(pf);
                i40e_irq_dynamic_enable_icr0(pf);
        }

        return IRQ_HANDLED;
}

/**
 * i40e_map_vector_to_qp - Assigns the queue pair to the vector
 * @vsi: the VSI being configured
 * @v_idx: vector index
 * @qp_idx: queue pair index
 **/
static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
{
        struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
        struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
        struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];

        tx_ring->q_vector = q_vector;
        tx_ring->next = q_vector->tx.ring;
        q_vector->tx.ring = tx_ring;
        q_vector->tx.count++;

        rx_ring->q_vector = q_vector;
        rx_ring->next = q_vector->rx.ring;
        q_vector->rx.ring = rx_ring;
        q_vector->rx.count++;
}

/**
 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
 * @vsi: the VSI being configured
 *
 * This function maps descriptor rings to the queue-specific vectors
 * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 * one vector per queue pair, but on a constrained vector budget, we
 * group the queue pairs as "efficiently" as possible.
 **/
static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
{
        int qp_remaining = vsi->num_queue_pairs;
        int q_vectors = vsi->num_q_vectors;
        int num_ringpairs;
        int v_start = 0;
        int qp_idx = 0;

        /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
         * group them so there are multiple queues per vector.
         */
        for (; v_start < q_vectors && qp_remaining; v_start++) {
                struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];

                num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);

                q_vector->num_ringpairs = num_ringpairs;

                q_vector->rx.count = 0;
                q_vector->tx.count = 0;
                q_vector->rx.ring = NULL;
                q_vector->tx.ring = NULL;

                while (num_ringpairs--) {
                        map_vector_to_qp(vsi, v_start, qp_idx);
                        qp_idx++;
                        qp_remaining--;
                }
        }
}

/**
 * i40e_vsi_request_irq - Request IRQ from the OS
 * @vsi: the VSI being configured
 * @basename: name for the vector
 **/
static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
{
        struct i40e_pf *pf = vsi->back;
        int err;

        if (pf->flags & I40E_FLAG_MSIX_ENABLED)
                err = i40e_vsi_request_irq_msix(vsi, basename);
        else if (pf->flags & I40E_FLAG_MSI_ENABLED)
                err = request_irq(pf->pdev->irq, i40e_intr, 0,
                                  pf->misc_int_name, pf);
        else
                err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
                                  pf->misc_int_name, pf);

        if (err)
                dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);

        return err;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
/**
 * i40e_netpoll - A Polling 'interrupt'handler
 * @netdev: network interface device structure
 *
 * This is used by netconsole to send skbs without having to re-enable
 * interrupts.  It's not called while the normal interrupt routine is executing.
 **/
static void i40e_netpoll(struct net_device *netdev)
{
        struct i40e_netdev_priv *np = netdev_priv(netdev);
        struct i40e_vsi *vsi = np->vsi;
        struct i40e_pf *pf = vsi->back;
        int i;

        /* if interface is down do nothing */
        if (test_bit(__I40E_DOWN, &vsi->state))
                return;

        pf->flags |= I40E_FLAG_IN_NETPOLL;
        if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
                for (i = 0; i < vsi->num_q_vectors; i++)
                        i40e_msix_clean_rings(0, vsi->q_vectors[i]);
        } else {
                i40e_intr(pf->pdev->irq, netdev);
        }
        pf->flags &= ~I40E_FLAG_IN_NETPOLL;
}
#endif

/**
 * i40e_vsi_control_tx - Start or stop a VSI's rings
 * @vsi: the VSI being configured
 * @enable: start or stop the rings
 **/
static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
{
        struct i40e_pf *pf = vsi->back;
        struct i40e_hw *hw = &pf->hw;
        int i, j, pf_q;
        u32 tx_reg;

        pf_q = vsi->base_queue;
        for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
                j = 1000;
                do {
                        usleep_range(1000, 2000);
                        tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
                } while (j-- && ((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT)
                               ^ (tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)) & 1);

                if (enable) {
                        /* is STAT set ? */
                        if ((tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) {
                                dev_info(&pf->pdev->dev,
                                         "Tx %d already enabled\n", i);
                                continue;
                        }
                } else {
                        /* is !STAT set ? */
                        if (!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) {
                                dev_info(&pf->pdev->dev,
                                         "Tx %d already disabled\n", i);
                                continue;
                        }
                }

                /* turn on/off the queue */
                if (enable)
                        tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK |
                                  I40E_QTX_ENA_QENA_STAT_MASK;
                else
                        tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;

                wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);