/*******************************************************************************

  Intel 10 Gigabit PCI Express Linux driver
  Copyright(c) 1999 - 2016 Intel Corporation.

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  Linux NICS <linux.nics@intel.com>
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

#include <linux/types.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/interrupt.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
#include <linux/pkt_sched.h>
#include <linux/ipv6.h>
#include <linux/slab.h>
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/if.h>
#include <linux/if_vlan.h>
#include <linux/if_macvlan.h>
#include <linux/if_bridge.h>
#include <linux/prefetch.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/atomic.h>
#include <scsi/fc/fc_fcoe.h>
#include <net/udp_tunnel.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>
#include <net/vxlan.h>
#include <net/mpls.h>

#include "ixgbe.h"
#include "ixgbe_common.h"
#include "ixgbe_dcb_82599.h"
#include "ixgbe_sriov.h"
#include "ixgbe_model.h"

char ixgbe_driver_name[] = "ixgbe";
static const char ixgbe_driver_string[] =
                              "Intel(R) 10 Gigabit PCI Express Network Driver";
#ifdef IXGBE_FCOE
char ixgbe_default_device_descr[] =
                              "Intel(R) 10 Gigabit Network Connection";
#else
static char ixgbe_default_device_descr[] =
                              "Intel(R) 10 Gigabit Network Connection";
#endif
#define DRV_VERSION "5.1.0-k"
const char ixgbe_driver_version[] = DRV_VERSION;
static const char ixgbe_copyright[] =
                                "Copyright (c) 1999-2016 Intel Corporation.";

static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";

static const struct ixgbe_info *ixgbe_info_tbl[] = {
        [board_82598]           = &ixgbe_82598_info,
        [board_82599]           = &ixgbe_82599_info,
        [board_X540]            = &ixgbe_X540_info,
        [board_X550]            = &ixgbe_X550_info,
        [board_X550EM_x]        = &ixgbe_X550EM_x_info,
        [board_x550em_x_fw]     = &ixgbe_x550em_x_fw_info,
        [board_x550em_a]        = &ixgbe_x550em_a_info,
        [board_x550em_a_fw]     = &ixgbe_x550em_a_fw_info,
};

/* ixgbe_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
static const struct pci_device_id ixgbe_pci_tbl[] = {
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
        /* required last entry */
        {0, }
};
MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);

#ifdef CONFIG_IXGBE_DCA
static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
                            void *p);
static struct notifier_block dca_notifier = {
        .notifier_call = ixgbe_notify_dca,
        .next          = NULL,
        .priority      = 0
};
#endif

#ifdef CONFIG_PCI_IOV
static unsigned int max_vfs;
module_param(max_vfs, uint, 0);
MODULE_PARM_DESC(max_vfs,
                 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
#endif /* CONFIG_PCI_IOV */

static unsigned int allow_unsupported_sfp;
module_param(allow_unsupported_sfp, uint, 0);
MODULE_PARM_DESC(allow_unsupported_sfp,
                 "Allow unsupported and untested SFP+ modules on 82599-based adapters");

#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

static struct workqueue_struct *ixgbe_wq;

static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);

static const struct net_device_ops ixgbe_netdev_ops;

static bool netif_is_ixgbe(struct net_device *dev)
{
        return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
}

static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
                                          u32 reg, u16 *value)
{
        struct pci_dev *parent_dev;
        struct pci_bus *parent_bus;

        parent_bus = adapter->pdev->bus->parent;
        if (!parent_bus)
                return -1;

        parent_dev = parent_bus->self;
        if (!parent_dev)
                return -1;

        if (!pci_is_pcie(parent_dev))
                return -1;

        pcie_capability_read_word(parent_dev, reg, value);
        if (*value == IXGBE_FAILED_READ_CFG_WORD &&
            ixgbe_check_cfg_remove(&adapter->hw, parent_dev))
                return -1;
        return 0;
}

static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u16 link_status = 0;
        int err;

        hw->bus.type = ixgbe_bus_type_pci_express;

        /* Get the negotiated link width and speed from PCI config space of the
         * parent, as this device is behind a switch
         */
        err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);

        /* assume caller will handle error case */
        if (err)
                return err;

        hw->bus.width = ixgbe_convert_bus_width(link_status);
        hw->bus.speed = ixgbe_convert_bus_speed(link_status);

        return 0;
}

/**
 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent
 * @hw: hw specific details
 *
 * This function is used by probe to determine whether a device's PCI-Express
 * bandwidth details should be gathered from the parent bus instead of from the
 * device. Used to ensure that various locations all have the correct device ID
 * checks.
 */
static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
{
        switch (hw->device_id) {
        case IXGBE_DEV_ID_82599_SFP_SF_QP:
        case IXGBE_DEV_ID_82599_QSFP_SF_QP:
                return true;
        default:
                return false;
        }
}

static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
                                     int expected_gts)
{
        struct ixgbe_hw *hw = &adapter->hw;
        int max_gts = 0;
        enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
        enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
        struct pci_dev *pdev;

        /* Some devices are not connected over PCIe and thus do not negotiate
         * speed. These devices do not have valid bus info, and thus any report
         * we generate may not be correct.
         */
        if (hw->bus.type == ixgbe_bus_type_internal)
                return;

        /* determine whether to use the parent device */
        if (ixgbe_pcie_from_parent(&adapter->hw))
                pdev = adapter->pdev->bus->parent->self;
        else
                pdev = adapter->pdev;

        if (pcie_get_minimum_link(pdev, &speed, &width) ||
            speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
                e_dev_warn("Unable to determine PCI Express bandwidth.\n");
                return;
        }

        switch (speed) {
        case PCIE_SPEED_2_5GT:
                /* 8b/10b encoding reduces max throughput by 20% */
                max_gts = 2 * width;
                break;
        case PCIE_SPEED_5_0GT:
                /* 8b/10b encoding reduces max throughput by 20% */
                max_gts = 4 * width;
                break;
        case PCIE_SPEED_8_0GT:
                /* 128b/130b encoding reduces throughput by less than 2% */
                max_gts = 8 * width;
                break;
        default:
                e_dev_warn("Unable to determine PCI Express bandwidth.\n");
                return;
        }

        e_dev_info("PCI Express bandwidth of %dGT/s available\n",
                   max_gts);
        e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n",
                   (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
                    speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
                    speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
                    "Unknown"),
                   width,
                   (speed == PCIE_SPEED_2_5GT ? "20%" :
                    speed == PCIE_SPEED_5_0GT ? "20%" :
                    speed == PCIE_SPEED_8_0GT ? "<2%" :
                    "Unknown"));

        if (max_gts < expected_gts) {
                e_dev_warn("This is not sufficient for optimal performance of this card.\n");
                e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n",
                        expected_gts);
                e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n");
        }
}

static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
{
        if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
            !test_bit(__IXGBE_REMOVING, &adapter->state) &&
            !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
                queue_work(ixgbe_wq, &adapter->service_task);
}

static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
{
        struct ixgbe_adapter *adapter = hw->back;

        if (!hw->hw_addr)
                return;
        hw->hw_addr = NULL;
        e_dev_err("Adapter removed\n");
        if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
                ixgbe_service_event_schedule(adapter);
}

static void ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
{
        u32 value;

        /* The following check not only optimizes a bit by not
         * performing a read on the status register when the
         * register just read was a status register read that
         * returned IXGBE_FAILED_READ_REG. It also blocks any
         * potential recursion.
         */
        if (reg == IXGBE_STATUS) {
                ixgbe_remove_adapter(hw);
                return;
        }
        value = ixgbe_read_reg(hw, IXGBE_STATUS);
        if (value == IXGBE_FAILED_READ_REG)
                ixgbe_remove_adapter(hw);
}

/**
 * ixgbe_read_reg - Read from device register
 * @hw: hw specific details
 * @reg: offset of register to read
 *
 * Returns : value read or IXGBE_FAILED_READ_REG if removed
 *
 * This function is used to read device registers. It checks for device
 * removal by confirming any read that returns all ones by checking the
 * status register value for all ones. This function avoids reading from
 * the hardware if a removal was previously detected in which case it
 * returns IXGBE_FAILED_READ_REG (all ones).
 */
u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
{
        u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
        u32 value;

        if (ixgbe_removed(reg_addr))
                return IXGBE_FAILED_READ_REG;
        if (unlikely(hw->phy.nw_mng_if_sel &
                     IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
                struct ixgbe_adapter *adapter;
                int i;

                for (i = 0; i < 200; ++i) {
                        value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY);
                        if (likely(!value))
                                goto writes_completed;
                        if (value == IXGBE_FAILED_READ_REG) {
                                ixgbe_remove_adapter(hw);
                                return IXGBE_FAILED_READ_REG;
                        }
                        udelay(5);
                }

                adapter = hw->back;
                e_warn(hw, "register writes incomplete %08x\n", value);
        }

writes_completed:
        value = readl(reg_addr + reg);
        if (unlikely(value == IXGBE_FAILED_READ_REG))
                ixgbe_check_remove(hw, reg);
        return value;
}

static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
{
        u16 value;

        pci_read_config_word(pdev, PCI_VENDOR_ID, &value);
        if (value == IXGBE_FAILED_READ_CFG_WORD) {
                ixgbe_remove_adapter(hw);
                return true;
        }
        return false;
}

u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
{
        struct ixgbe_adapter *adapter = hw->back;
        u16 value;

        if (ixgbe_removed(hw->hw_addr))
                return IXGBE_FAILED_READ_CFG_WORD;
        pci_read_config_word(adapter->pdev, reg, &value);
        if (value == IXGBE_FAILED_READ_CFG_WORD &&
            ixgbe_check_cfg_remove(hw, adapter->pdev))
                return IXGBE_FAILED_READ_CFG_WORD;
        return value;
}

#ifdef CONFIG_PCI_IOV
static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
{
        struct ixgbe_adapter *adapter = hw->back;
        u32 value;

        if (ixgbe_removed(hw->hw_addr))
                return IXGBE_FAILED_READ_CFG_DWORD;
        pci_read_config_dword(adapter->pdev, reg, &value);
        if (value == IXGBE_FAILED_READ_CFG_DWORD &&
            ixgbe_check_cfg_remove(hw, adapter->pdev))
                return IXGBE_FAILED_READ_CFG_DWORD;
        return value;
}
#endif /* CONFIG_PCI_IOV */

void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
{
        struct ixgbe_adapter *adapter = hw->back;

        if (ixgbe_removed(hw->hw_addr))
                return;
        pci_write_config_word(adapter->pdev, reg, value);
}

static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
{
        BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));

        /* flush memory to make sure state is correct before next watchdog */
        smp_mb__before_atomic();
        clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
}

struct ixgbe_reg_info {
        u32 ofs;
        char *name;
};

static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {

        /* General Registers */
        {IXGBE_CTRL, "CTRL"},
        {IXGBE_STATUS, "STATUS"},
        {IXGBE_CTRL_EXT, "CTRL_EXT"},

        /* Interrupt Registers */
        {IXGBE_EICR, "EICR"},

        /* RX Registers */
        {IXGBE_SRRCTL(0), "SRRCTL"},
        {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
        {IXGBE_RDLEN(0), "RDLEN"},
        {IXGBE_RDH(0), "RDH"},
        {IXGBE_RDT(0), "RDT"},
        {IXGBE_RXDCTL(0), "RXDCTL"},
        {IXGBE_RDBAL(0), "RDBAL"},
        {IXGBE_RDBAH(0), "RDBAH"},

        /* TX Registers */
        {IXGBE_TDBAL(0), "TDBAL"},
        {IXGBE_TDBAH(0), "TDBAH"},
        {IXGBE_TDLEN(0), "TDLEN"},
        {IXGBE_TDH(0), "TDH"},
        {IXGBE_TDT(0), "TDT"},
        {IXGBE_TXDCTL(0), "TXDCTL"},

        /* List Terminator */
        { .name = NULL }
};


/*
 * ixgbe_regdump - register printout routine
 */
static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
{
        int i;
        char rname[16];
        u32 regs[64];

        switch (reginfo->ofs) {
        case IXGBE_SRRCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
                break;
        case IXGBE_DCA_RXCTRL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
                break;
        case IXGBE_RDLEN(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
                break;
        case IXGBE_RDH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
                break;
        case IXGBE_RDT(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
                break;
        case IXGBE_RXDCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
                break;
        case IXGBE_RDBAL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
                break;
        case IXGBE_RDBAH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
                break;
        case IXGBE_TDBAL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
                break;
        case IXGBE_TDBAH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
                break;
        case IXGBE_TDLEN(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
                break;
        case IXGBE_TDH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
                break;
        case IXGBE_TDT(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
                break;
        case IXGBE_TXDCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
                break;
        default:
                pr_info("%-15s %08x\n",
                        reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));
                return;
        }

        i = 0;
        while (i < 64) {
                int j;
                char buf[9 * 8 + 1];
                char *p = buf;

                snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i, i + 7);
                for (j = 0; j < 8; j++)
                        p += sprintf(p, " %08x", regs[i++]);
                pr_err("%-15s%s\n", rname, buf);
        }

}

static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)
{
        struct ixgbe_tx_buffer *tx_buffer;

        tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];
        pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
                n, ring->next_to_use, ring->next_to_clean,
                (u64)dma_unmap_addr(tx_buffer, dma),
                dma_unmap_len(tx_buffer, len),
                tx_buffer->next_to_watch,
                (u64)tx_buffer->time_stamp);
}

/*
 * ixgbe_dump - Print registers, tx-rings and rx-rings
 */
static void ixgbe_dump(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_reg_info *reginfo;
        int n = 0;
        struct ixgbe_ring *ring;
        struct ixgbe_tx_buffer *tx_buffer;
        union ixgbe_adv_tx_desc *tx_desc;
        struct my_u0 { u64 a; u64 b; } *u0;
        struct ixgbe_ring *rx_ring;
        union ixgbe_adv_rx_desc *rx_desc;
        struct ixgbe_rx_buffer *rx_buffer_info;
        int i = 0;

        if (!netif_msg_hw(adapter))
                return;

        /* Print netdevice Info */
        if (netdev) {
                dev_info(&adapter->pdev->dev, "Net device Info\n");
                pr_info("Device Name     state            "
                        "trans_start\n");
                pr_info("%-15s %016lX %016lX\n",
                        netdev->name,
                        netdev->state,
                        dev_trans_start(netdev));
        }

        /* Print Registers */
        dev_info(&adapter->pdev->dev, "Register Dump\n");
        pr_info(" Register Name   Value\n");
        for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
             reginfo->name; reginfo++) {
                ixgbe_regdump(hw, reginfo);
        }

        /* Print TX Ring Summary */
        if (!netdev || !netif_running(netdev))
                return;

        dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
        pr_info(" %s     %s              %s        %s\n",
                "Queue [NTU] [NTC] [bi(ntc)->dma  ]",
                "leng", "ntw", "timestamp");
        for (n = 0; n < adapter->num_tx_queues; n++) {
                ring = adapter->tx_ring[n];
                ixgbe_print_buffer(ring, n);
        }

        for (n = 0; n < adapter->num_xdp_queues; n++) {
                ring = adapter->xdp_ring[n];
                ixgbe_print_buffer(ring, n);
        }

        /* Print TX Rings */
        if (!netif_msg_tx_done(adapter))
                goto rx_ring_summary;

        dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

        /* Transmit Descriptor Formats
         *
         * 82598 Advanced Transmit Descriptor
         *   +--------------------------------------------------------------+
         * 0 |         Buffer Address [63:0]                                |
         *   +--------------------------------------------------------------+
         * 8 |  PAYLEN  | POPTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
         *   +--------------------------------------------------------------+
         *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
         *
         * 82598 Advanced Transmit Descriptor (Write-Back Format)
         *   +--------------------------------------------------------------+
         * 0 |                          RSV [63:0]                          |
         *   +--------------------------------------------------------------+
         * 8 |            RSV           |  STA  |          NXTSEQ           |
         *   +--------------------------------------------------------------+
         *   63                       36 35   32 31                         0
         *
         * 82599+ Advanced Transmit Descriptor
         *   +--------------------------------------------------------------+
         * 0 |         Buffer Address [63:0]                                |
         *   +--------------------------------------------------------------+
         * 8 |PAYLEN  |POPTS|CC|IDX  |STA  |DCMD  |DTYP |MAC  |RSV  |DTALEN |
         *   +--------------------------------------------------------------+
         *   63     46 45 40 39 38 36 35 32 31  24 23 20 19 18 17 16 15     0
         *
         * 82599+ Advanced Transmit Descriptor (Write-Back Format)
         *   +--------------------------------------------------------------+
         * 0 |                          RSV [63:0]                          |
         *   +--------------------------------------------------------------+
         * 8 |            RSV           |  STA  |           RSV             |
         *   +--------------------------------------------------------------+
         *   63                       36 35   32 31                         0
         */

        for (n = 0; n < adapter->num_tx_queues; n++) {
                ring = adapter->tx_ring[n];
                pr_info("------------------------------------\n");
                pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);
                pr_info("------------------------------------\n");
                pr_info("%s%s    %s              %s        %s          %s\n",
                        "T [desc]     [address 63:0  ] ",
                        "[PlPOIdStDDt Ln] [bi->dma       ] ",
                        "leng", "ntw", "timestamp", "bi->skb");

                for (i = 0; ring->desc && (i < ring->count); i++) {
                        tx_desc = IXGBE_TX_DESC(ring, i);
                        tx_buffer = &ring->tx_buffer_info[i];
                        u0 = (struct my_u0 *)tx_desc;
                        if (dma_unmap_len(tx_buffer, len) > 0) {
                                const char *ring_desc;

                                if (i == ring->next_to_use &&
                                    i == ring->next_to_clean)
                                        ring_desc = " NTC/U";
                                else if (i == ring->next_to_use)
                                        ring_desc = " NTU";
                                else if (i == ring->next_to_clean)
                                        ring_desc = " NTC";
                                else
                                        ring_desc = "";
                                pr_info("T [0x%03X]    %016llX %016llX %016llX %08X %p %016llX %p%s",
                                        i,
                                        le64_to_cpu(u0->a),
                                        le64_to_cpu(u0->b),
                                        (u64)dma_unmap_addr(tx_buffer, dma),
                                        dma_unmap_len(tx_buffer, len),
                                        tx_buffer->next_to_watch,
                                        (u64)tx_buffer->time_stamp,
                                        tx_buffer->skb,
                                        ring_desc);

                                if (netif_msg_pktdata(adapter) &&
                                    tx_buffer->skb)
                                        print_hex_dump(KERN_INFO, "",
                                                DUMP_PREFIX_ADDRESS, 16, 1,
                                                tx_buffer->skb->data,
                                                dma_unmap_len(tx_buffer, len),
                                                true);
                        }
                }
        }

        /* Print RX Rings Summary */
rx_ring_summary:
        dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
        pr_info("Queue [NTU] [NTC]\n");
        for (n = 0; n < adapter->num_rx_queues; n++) {
                rx_ring = adapter->rx_ring[n];
                pr_info("%5d %5X %5X\n",
                        n, rx_ring->next_to_use, rx_ring->next_to_clean);
        }

        /* Print RX Rings */
        if (!netif_msg_rx_status(adapter))
                return;

        dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

        /* Receive Descriptor Formats
         *
         * 82598 Advanced Receive Descriptor (Read) Format
         *    63                                           1        0
         *    +-----------------------------------------------------+
         *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
         *    +----------------------------------------------+------+
         *  8 |       Header Buffer Address [63:1]           |  DD  |
         *    +-----------------------------------------------------+
         *
         *
         * 82598 Advanced Receive Descriptor (Write-Back) Format
         *
         *   63       48 47    32 31  30      21 20 16 15   4 3     0
         *   +------------------------------------------------------+
         * 0 |       RSS Hash /  |SPH| HDR_LEN  | RSV |Packet|  RSS |
         *   | Packet   | IP     |   |          |     | Type | Type |
         *   | Checksum | Ident  |   |          |     |      |      |
         *   +------------------------------------------------------+
         * 8 | VLAN Tag | Length | Extended Error | Extended Status |
         *   +------------------------------------------------------+
         *   63       48 47    32 31            20 19               0
         *
         * 82599+ Advanced Receive Descriptor (Read) Format
         *    63                                           1        0
         *    +-----------------------------------------------------+
         *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
         *    +----------------------------------------------+------+
         *  8 |       Header Buffer Address [63:1]           |  DD  |
         *    +-----------------------------------------------------+
         *
         *
         * 82599+ Advanced Receive Descriptor (Write-Back) Format
         *
         *   63       48 47    32 31  30      21 20 17 16   4 3     0
         *   +------------------------------------------------------+
         * 0 |RSS / Frag Checksum|SPH| HDR_LEN  |RSC- |Packet|  RSS |
         *   |/ RTT / PCoE_PARAM |   |          | CNT | Type | Type |
         *   |/ Flow Dir Flt ID  |   |          |     |      |      |
         *   +------------------------------------------------------+
         * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
         *   +------------------------------------------------------+
         *   63       48 47    32 31          20 19                 0
         */

        for (n = 0; n < adapter->num_rx_queues; n++) {
                rx_ring = adapter->rx_ring[n];
                pr_info("------------------------------------\n");
                pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
                pr_info("------------------------------------\n");
                pr_info("%s%s%s\n",
                        "R  [desc]      [ PktBuf     A0] ",
                        "[  HeadBuf   DD] [bi->dma       ] [bi->skb       ] ",
                        "<-- Adv Rx Read format");
                pr_info("%s%s%s\n",
                        "RWB[desc]      [PcsmIpSHl PtRs] ",
                        "[vl er S cks ln] ---------------- [bi->skb       ] ",
                        "<-- Adv Rx Write-Back format");

                for (i = 0; i < rx_ring->count; i++) {
                        const char *ring_desc;

                        if (i == rx_ring->next_to_use)
                                ring_desc = " NTU";
                        else if (i == rx_ring->next_to_clean)
                                ring_desc = " NTC";
                        else
                                ring_desc = "";

                        rx_buffer_info = &rx_ring->rx_buffer_info[i];
                        rx_desc = IXGBE_RX_DESC(rx_ring, i);
                        u0 = (struct my_u0 *)rx_desc;
                        if (rx_desc->wb.upper.length) {
                                /* Descriptor Done */
                                pr_info("RWB[0x%03X]     %016llX %016llX ---------------- %p%s\n",
                                        i,
                                        le64_to_cpu(u0->a),
                                        le64_to_cpu(u0->b),
                                        rx_buffer_info->skb,
                                        ring_desc);
                        } else {
                                pr_info("R  [0x%03X]     %016llX %016llX %016llX %p%s\n",
                                        i,
                                        le64_to_cpu(u0->a),
                                        le64_to_cpu(u0->b),
                                        (u64)rx_buffer_info->dma,
                                        rx_buffer_info->skb,
                                        ring_desc);

                                if (netif_msg_pktdata(adapter) &&
                                    rx_buffer_info->dma) {
                                        print_hex_dump(KERN_INFO, "",
                                           DUMP_PREFIX_ADDRESS, 16, 1,
                                           page_address(rx_buffer_info->page) +
                                                    rx_buffer_info->page_offset,
                                           ixgbe_rx_bufsz(rx_ring), true);
                                }
                        }
                }
        }
}

static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
{
        u32 ctrl_ext;

        /* Let firmware take over control of h/w */
        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
                        ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
}

static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
{
        u32 ctrl_ext;

        /* Let firmware know the driver has taken over */
        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
                        ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
}

/**
 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 * @adapter: pointer to adapter struct
 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 * @queue: queue to map the corresponding interrupt to
 * @msix_vector: the vector to map to the corresponding queue
 *
 */
static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
                           u8 queue, u8 msix_vector)
{
        u32 ivar, index;
        struct ixgbe_hw *hw = &adapter->hw;
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                if (direction == -1)
                        direction = 0;
                index = (((direction * 64) + queue) >> 2) & 0x1F;
                ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
                ivar &= ~(0xFF << (8 * (queue & 0x3)));
                ivar |= (msix_vector << (8 * (queue & 0x3)));
                IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (direction == -1) {
                        /* other causes */
                        msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                        index = ((queue & 1) * 8);
                        ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
                        ivar &= ~(0xFF << index);
                        ivar |= (msix_vector << index);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
                        break;
                } else {
                        /* tx or rx causes */
                        msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                        index = ((16 * (queue & 1)) + (8 * direction));
                        ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
                        ivar &= ~(0xFF << index);
                        ivar |= (msix_vector << index);
                        IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
                        break;
                }
        default:
                break;
        }
}

static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
                                          u64 qmask)
{
        u32 mask;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                mask = (qmask & 0xFFFFFFFF);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
                mask = (qmask >> 32);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
                break;
        default:
                break;
        }
}

static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_hw_stats *hwstats = &adapter->stats;
        int i;
        u32 data;

        if ((hw->fc.current_mode != ixgbe_fc_full) &&
            (hw->fc.current_mode != ixgbe_fc_rx_pause))
                return;

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
                break;
        default:
                data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
        }
        hwstats->lxoffrxc += data;

        /* refill credits (no tx hang) if we received xoff */
        if (!data)
                return;

        for (i = 0; i < adapter->num_tx_queues; i++)
                clear_bit(__IXGBE_HANG_CHECK_ARMED,
                          &adapter->tx_ring[i]->state);

        for (i = 0; i < adapter->num_xdp_queues; i++)
                clear_bit(__IXGBE_HANG_CHECK_ARMED,
                          &adapter->xdp_ring[i]->state);
}

static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_hw_stats *hwstats = &adapter->stats;
        u32 xoff[8] = {0};
        u8 tc;
        int i;
        bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

        if (adapter->ixgbe_ieee_pfc)
                pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

        if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
                ixgbe_update_xoff_rx_lfc(adapter);
                return;
        }

        /* update stats for each tc, only valid with PFC enabled */
        for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
                u32 pxoffrxc;

                switch (hw->mac.type) {
                case ixgbe_mac_82598EB:
                        pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
                        break;
                default:
                        pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
                }
                hwstats->pxoffrxc[i] += pxoffrxc;
                /* Get the TC for given UP */
                tc = netdev_get_prio_tc_map(adapter->netdev, i);
                xoff[tc] += pxoffrxc;
        }

        /* disarm tx queues that have received xoff frames */
        for (i = 0; i < adapter->num_tx_queues; i++) {
                struct ixgbe_ring *tx_ring = adapter->tx_ring[i];

                tc = tx_ring->dcb_tc;
                if (xoff[tc])
                        clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
        }

        for (i = 0; i < adapter->num_xdp_queues; i++) {
                struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];

                tc = xdp_ring->dcb_tc;
                if (xoff[tc])
                        clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);
        }
}

static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
{
        return ring->stats.packets;
}

static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
{
        unsigned int head, tail;

        head = ring->next_to_clean;
        tail = ring->next_to_use;

        return ((head <= tail) ? tail : tail + ring->count) - head;
}

static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
{
        u32 tx_done = ixgbe_get_tx_completed(tx_ring);
        u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
        u32 tx_pending = ixgbe_get_tx_pending(tx_ring);

        clear_check_for_tx_hang(tx_ring);

        /*
         * Check for a hung queue, but be thorough. This verifies
         * that a transmit has been completed since the previous
         * check AND there is at least one packet pending. The
         * ARMED bit is set to indicate a potential hang. The
         * bit is cleared if a pause frame is received to remove
         * false hang detection due to PFC or 802.3x frames. By
         * requiring this to fail twice we avoid races with
         * pfc clearing the ARMED bit and conditions where we
         * run the check_tx_hang logic with a transmit completion
         * pending but without time to complete it yet.
         */
        if (tx_done_old == tx_done && tx_pending)
                /* make sure it is true for two checks in a row */
                return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
                                        &tx_ring->state);
        /* update completed stats and continue */
        tx_ring->tx_stats.tx_done_old = tx_done;
        /* reset the countdown */
        clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);

        return false;
}

/**
 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 * @adapter: driver private struct
 **/
static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
{

        /* Do the reset outside of interrupt context */
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                e_warn(drv, "initiating reset due to tx timeout\n");
                ixgbe_service_event_schedule(adapter);
        }
}

/**
 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
 * @netdev: network interface device structure
 * @queue_index: Tx queue to set
 * @maxrate: desired maximum transmit bitrate
 **/
static int ixgbe_tx_maxrate(struct net_device *netdev,
                            int queue_index, u32 maxrate)
{
        struct ixgbe_adapter *adapter = netdev_priv(netdev);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 bcnrc_val = ixgbe_link_mbps(adapter);

        if (!maxrate)
                return 0;

        /* Calculate the rate factor values to set */
        bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
        bcnrc_val /= maxrate;

        /* clear everything but the rate factor */
        bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
        IXGBE_RTTBCNRC_RF_DEC_MASK;

        /* enable the rate scheduler */
        bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;

        IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
        IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);

        return 0;
}

/**
 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
 * @q_vector: structure containing interrupt and ring information
 * @tx_ring: tx ring to clean
 * @napi_budget: Used to determine if we are in netpoll
 **/
static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
                               struct ixgbe_ring *tx_ring, int napi_budget)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_tx_buffer *tx_buffer;
        union ixgbe_adv_tx_desc *tx_desc;
        unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
        unsigned int budget = q_vector->tx.work_limit;
        unsigned int i = tx_ring->next_to_clean;

        if (test_bit(__IXGBE_DOWN, &adapter->state))
                return true;

        tx_buffer = &tx_ring->tx_buffer_info[i];
        tx_desc = IXGBE_TX_DESC(tx_ring, i);
        i -= tx_ring->count;

        do {
                union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;

                /* if next_to_watch is not set then there is no work pending */
                if (!eop_desc)
                        break;

                /* prevent any other reads prior to eop_desc */
                smp_rmb();

                /* if DD is not set pending work has not been completed */
                if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
                        break;

                /* clear next_to_watch to prevent false hangs */
                tx_buffer->next_to_watch = NULL;

                /* update the statistics for this packet */
                total_bytes += tx_buffer->bytecount;
                total_packets += tx_buffer->gso_segs;
                if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
                        total_ipsec++;

                /* free the skb */
                if (ring_is_xdp(tx_ring))
                        page_frag_free(tx_buffer->data);
                else
                        napi_consume_skb(tx_buffer->skb, napi_budget);

                /* unmap skb header data */
                dma_unmap_single(tx_ring->dev,
                                 dma_unmap_addr(tx_buffer, dma),
                                 dma_unmap_len(tx_buffer, len),
                                 DMA_TO_DEVICE);

                /* clear tx_buffer data */
                dma_unmap_len_set(tx_buffer, len, 0);

                /* unmap remaining buffers */
                while (tx_desc != eop_desc) {
                        tx_buffer++;
                        tx_desc++;
                        i++;
                        if (unlikely(!i)) {
                                i -= tx_ring->count;
                                tx_buffer = tx_ring->tx_buffer_info;
                                tx_desc = IXGBE_TX_DESC(tx_ring, 0);
                        }

                        /* unmap any remaining paged data */
                        if (dma_unmap_len(tx_buffer, len)) {
                                dma_unmap_page(tx_ring->dev,
                                               dma_unmap_addr(tx_buffer, dma),
                                               dma_unmap_len(tx_buffer, len),
                                               DMA_TO_DEVICE);
                                dma_unmap_len_set(tx_buffer, len, 0);
                        }
                }

                /* move us one more past the eop_desc for start of next pkt */
                tx_buffer++;
                tx_desc++;
                i++;
                if (unlikely(!i)) {
                        i -= tx_ring->count;
                        tx_buffer = tx_ring->tx_buffer_info;
                        tx_desc = IXGBE_TX_DESC(tx_ring, 0);
                }

                /* issue prefetch for next Tx descriptor */
                prefetch(tx_desc);

                /* update budget accounting */
                budget--;
        } while (likely(budget));

        i += tx_ring->count;
        tx_ring->next_to_clean = i;
        u64_stats_update_begin(&tx_ring->syncp);
        tx_ring->stats.bytes += total_bytes;
        tx_ring->stats.packets += total_packets;
        u64_stats_update_end(&tx_ring->syncp);
        q_vector->tx.total_bytes += total_bytes;
        q_vector->tx.total_packets += total_packets;
        adapter->tx_ipsec += total_ipsec;

        if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
                /* schedule immediate reset if we believe we hung */
                struct ixgbe_hw *hw = &adapter->hw;
                e_err(drv, "Detected Tx Unit Hang %s\n"
                        "  Tx Queue             <%d>\n"
                        "  TDH, TDT             <%x>, <%x>\n"
                        "  next_to_use          <%x>\n"
                        "  next_to_clean        <%x>\n"
                        "tx_buffer_info[next_to_clean]\n"
                        "  time_stamp           <%lx>\n"
                        "  jiffies              <%lx>\n",
                        ring_is_xdp(tx_ring) ? "(XDP)" : "",
                        tx_ring->queue_index,
                        IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
                        IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
                        tx_ring->next_to_use, i,
                        tx_ring->tx_buffer_info[i].time_stamp, jiffies);

                if (!ring_is_xdp(tx_ring))
                        netif_stop_subqueue(tx_ring->netdev,
                                            tx_ring->queue_index);

                e_info(probe,
                       "tx hang %d detected on queue %d, resetting adapter\n",
                        adapter->tx_timeout_count + 1, tx_ring->queue_index);

                /* schedule immediate reset if we believe we hung */
                ixgbe_tx_timeout_reset(adapter);

                /* the adapter is about to reset, no point in enabling stuff */
                return true;
        }

        if (ring_is_xdp(tx_ring))
                return !!budget;

        netdev_tx_completed_queue(txring_txq(tx_ring),
                                  total_packets, total_bytes);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
        if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
                     (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
                /* Make sure that anybody stopping the queue after this
                 * sees the new next_to_clean.
                 */
                smp_mb();
                if (__netif_subqueue_stopped(tx_ring->netdev,
                                             tx_ring->queue_index)
                    && !test_bit(__IXGBE_DOWN, &adapter->state)) {
                        netif_wake_subqueue(tx_ring->netdev,
                                            tx_ring->queue_index);
                        ++tx_ring->tx_stats.restart_queue;
                }
        }

        return !!budget;
}

#ifdef CONFIG_IXGBE_DCA
static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
                                struct ixgbe_ring *tx_ring,
                                int cpu)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 txctrl = 0;
        u16 reg_offset;

        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                txctrl = dca3_get_tag(tx_ring->dev, cpu);

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
                reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
                txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
                break;
        default:
                /* for unknown hardware do not write register */
                return;
        }

        /*
         * We can enable relaxed ordering for reads, but not writes when
         * DCA is enabled.  This is due to a known issue in some chipsets
         * which will cause the DCA tag to be cleared.
         */
        txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
                  IXGBE_DCA_TXCTRL_DATA_RRO_EN |
                  IXGBE_DCA_TXCTRL_DESC_DCA_EN;

        IXGBE_WRITE_REG(hw, reg_offset, txctrl);
}

static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
                                struct ixgbe_ring *rx_ring,
                                int cpu)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 rxctrl = 0;
        u8 reg_idx = rx_ring->reg_idx;

        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                rxctrl = dca3_get_tag(rx_ring->dev, cpu);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
                rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
                break;
        default:
                break;
        }

        /*
         * We can enable relaxed ordering for reads, but not writes when
         * DCA is enabled.  This is due to a known issue in some chipsets
         * which will cause the DCA tag to be cleared.
         */
        rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
                  IXGBE_DCA_RXCTRL_DATA_DCA_EN |
                  IXGBE_DCA_RXCTRL_DESC_DCA_EN;

        IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
}

static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_ring *ring;
        int cpu = get_cpu();

        if (q_vector->cpu == cpu)
                goto out_no_update;

        ixgbe_for_each_ring(ring, q_vector->tx)
                ixgbe_update_tx_dca(adapter, ring, cpu);

        ixgbe_for_each_ring(ring, q_vector->rx)
                ixgbe_update_rx_dca(adapter, ring, cpu);

        q_vector->cpu = cpu;
out_no_update:
        put_cpu();
}

static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
{
        int i;

        /* always use CB2 mode, difference is masked in the CB driver */
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                IXGBE_DCA_CTRL_DCA_MODE_CB2);
        else
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                IXGBE_DCA_CTRL_DCA_DISABLE);

        for (i = 0; i < adapter->num_q_vectors; i++) {
                adapter->q_vector[i]->cpu = -1;
                ixgbe_update_dca(adapter->q_vector[i]);
        }
}

static int __ixgbe_notify_dca(struct device *dev, void *data)
{
        struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
        unsigned long event = *(unsigned long *)data;

        if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
                return 0;

        switch (event) {
        case DCA_PROVIDER_ADD:
                /* if we're already enabled, don't do it again */
                if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                        break;
                if (dca_add_requester(dev) == 0) {
                        adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                        IXGBE_DCA_CTRL_DCA_MODE_CB2);
                        break;
                }
                /* fall through - DCA is disabled. */
        case DCA_PROVIDER_REMOVE:
                if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
                        dca_remove_requester(dev);
                        adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                        IXGBE_DCA_CTRL_DCA_DISABLE);
                }
                break;
        }

        return 0;
}

#endif /* CONFIG_IXGBE_DCA */

#define IXGBE_RSS_L4_TYPES_MASK \
        ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))

static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
                                 union ixgbe_adv_rx_desc *rx_desc,
                                 struct sk_buff *skb)
{
        u16 rss_type;

        if (!(ring->netdev->features & NETIF_F_RXHASH))
                return;

        rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
                   IXGBE_RXDADV_RSSTYPE_MASK;

        if (!rss_type)
                return;

        skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
                     (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
                     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
}

#ifdef IXGBE_FCOE
/**
 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
 * @ring: structure containing ring specific data
 * @rx_desc: advanced rx descriptor
 *
 * Returns : true if it is FCoE pkt
 */
static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
                                    union ixgbe_adv_rx_desc *rx_desc)
{
        __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

        return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
               ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
                (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
                             IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
}

#endif /* IXGBE_FCOE */
/**
 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
 * @ring: structure containing ring specific data
 * @rx_desc: current Rx descriptor being processed
 * @skb: skb currently being received and modified
 **/
static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
                                     union ixgbe_adv_rx_desc *rx_desc,
                                     struct sk_buff *skb)
{
        __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
        bool encap_pkt = false;

        skb_checksum_none_assert(skb);

        /* Rx csum disabled */
        if (!(ring->netdev->features & NETIF_F_RXCSUM))
                return;

        /* check for VXLAN and Geneve packets */
        if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
                encap_pkt = true;
                skb->encapsulation = 1;
        }

        /* if IP and error */
        if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
            ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
                ring->rx_stats.csum_err++;
                return;
        }

        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
                return;

        if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
                /*
                 * 82599 errata, UDP frames with a 0 checksum can be marked as
                 * checksum errors.
                 */
                if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
                    test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
                        return;

                ring->rx_stats.csum_err++;
                return;
        }

        /* It must be a TCP or UDP packet with a valid checksum */
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        if (encap_pkt) {
                if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
                        return;

                if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
                        skb->ip_summed = CHECKSUM_NONE;
                        return;
                }
                /* If we checked the outer header let the stack know */
                skb->csum_level = 1;
        }
}

static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring)
{
        return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0;
}

static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
                                    struct ixgbe_rx_buffer *bi)
{
        struct page *page = bi->page;
        dma_addr_t dma;

        /* since we are recycling buffers we should seldom need to alloc */
        if (likely(page))
                return true;

        /* alloc new page for storage */
        page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
        if (unlikely(!page)) {
                rx_ring->rx_stats.alloc_rx_page_failed++;
                return false;
        }

        /* map page for use */
        dma = dma_map_page_attrs(rx_ring->dev, page, 0,
                                 ixgbe_rx_pg_size(rx_ring),
                                 DMA_FROM_DEVICE,
                                 IXGBE_RX_DMA_ATTR);

        /*
         * if mapping failed free memory back to system since
         * there isn't much point in holding memory we can't use
         */
        if (dma_mapping_error(rx_ring->dev, dma)) {
                __free_pages(page, ixgbe_rx_pg_order(rx_ring));

                rx_ring->rx_stats.alloc_rx_page_failed++;
                return false;
        }

        bi->dma = dma;
        bi->page = page;
        bi->page_offset = ixgbe_rx_offset(rx_ring);
        bi->pagecnt_bias = 1;
        rx_ring->rx_stats.alloc_rx_page++;

        return true;
}

/**
 * ixgbe_alloc_rx_buffers - Replace used receive buffers
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
 **/
void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
{
        union ixgbe_adv_rx_desc *rx_desc;
        struct ixgbe_rx_buffer *bi;
        u16 i = rx_ring->next_to_use;
        u16 bufsz;

        /* nothing to do */
        if (!cleaned_count)
                return;

        rx_desc = IXGBE_RX_DESC(rx_ring, i);
        bi = &rx_ring->rx_buffer_info[i];
        i -= rx_ring->count;

        bufsz = ixgbe_rx_bufsz(rx_ring);

        do {
                if (!ixgbe_alloc_mapped_page(rx_ring, bi))
                        break;

                /* sync the buffer for use by the device */
                dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
                                                 bi->page_offset, bufsz,
                                                 DMA_FROM_DEVICE);

                /*
                 * Refresh the desc even if buffer_addrs didn't change
                 * because each write-back erases this info.
                 */
                rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);

                rx_desc++;
                bi++;
                i++;
                if (unlikely(!i)) {
                        rx_desc = IXGBE_RX_DESC(rx_ring, 0);
                        bi = rx_ring->rx_buffer_info;
                        i -= rx_ring->count;
                }

                /* clear the length for the next_to_use descriptor */
                rx_desc->wb.upper.length = 0;

                cleaned_count--;
        } while (cleaned_count);

        i += rx_ring->count;

        if (rx_ring->next_to_use != i) {
                rx_ring->next_to_use = i;

                /* update next to alloc since we have filled the ring */
                rx_ring->next_to_alloc = i;

                /* Force memory writes to complete before letting h/w
                 * know there are new descriptors to fetch.  (Only
                 * applicable for weak-ordered memory model archs,
                 * such as IA-64).
                 */
                wmb();
                writel(i, rx_ring->tail);
        }
}

static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
                                   struct sk_buff *skb)
{
        u16 hdr_len = skb_headlen(skb);

        /* set gso_size to avoid messing up TCP MSS */
        skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
                                                 IXGBE_CB(skb)->append_cnt);
        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
}

static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
                                   struct sk_buff *skb)
{
        /* if append_cnt is 0 then frame is not RSC */
        if (!IXGBE_CB(skb)->append_cnt)
                return;

        rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
        rx_ring->rx_stats.rsc_flush++;

        ixgbe_set_rsc_gso_size(rx_ring, skb);

        /* gso_size is computed using append_cnt so always clear it last */
        IXGBE_CB(skb)->append_cnt = 0;
}

/**
 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being populated
 *
 * This function checks the ring, descriptor, and packet information in
 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
 * other fields within the skb.
 **/
static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
                                     union ixgbe_adv_rx_desc *rx_desc,
                                     struct sk_buff *skb)
{
        struct net_device *dev = rx_ring->netdev;
        u32 flags = rx_ring->q_vector->adapter->flags;

        ixgbe_update_rsc_stats(rx_ring, skb);

        ixgbe_rx_hash(rx_ring, rx_desc, skb);

        ixgbe_rx_checksum(rx_ring, rx_desc, skb);

        if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
                ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);

        if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
            ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
                u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
        }

        if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
                ixgbe_ipsec_rx(rx_ring, rx_desc, skb);

        skb->protocol = eth_type_trans(skb, dev);

        /* record Rx queue, or update MACVLAN statistics */
        if (netif_is_ixgbe(dev))
                skb_record_rx_queue(skb, rx_ring->queue_index);
        else
                macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
                                 (skb->pkt_type == PACKET_BROADCAST) ||
                                 (skb->pkt_type == PACKET_MULTICAST));
}

static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
                         struct sk_buff *skb)
{
        napi_gro_receive(&q_vector->napi, skb);
}

/**
 * ixgbe_is_non_eop - process handling of non-EOP buffers
 * @rx_ring: Rx ring being processed
 * @rx_desc: Rx descriptor for current buffer
 * @skb: Current socket buffer containing buffer in progress
 *
 * This function updates next to clean.  If the buffer is an EOP buffer
 * this function exits returning false, otherwise it will place the
 * sk_buff in the next buffer to be chained and return true indicating
 * that this is in fact a non-EOP buffer.
 **/
static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
                             union ixgbe_adv_rx_desc *rx_desc,
                             struct sk_buff *skb)
{
        u32 ntc = rx_ring->next_to_clean + 1;

        /* fetch, update, and store next to clean */
        ntc = (ntc < rx_ring->count) ? ntc : 0;
        rx_ring->next_to_clean = ntc;

        prefetch(IXGBE_RX_DESC(rx_ring, ntc));

        /* update RSC append count if present */
        if (ring_is_rsc_enabled(rx_ring)) {
                __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
                                     cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);

                if (unlikely(rsc_enabled)) {
                        u32 rsc_cnt = le32_to_cpu(rsc_enabled);

                        rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
                        IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;

                        /* update ntc based on RSC value */
                        ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
                        ntc &= IXGBE_RXDADV_NEXTP_MASK;
                        ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
                }
        }

        /* if we are the last buffer then there is nothing else to do */
        if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
                return false;

        /* place skb in next buffer to be received */
        rx_ring->rx_buffer_info[ntc].skb = skb;
        rx_ring->rx_stats.non_eop_descs++;

        return true;
}

/**
 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being adjusted
 *
 * This function is an ixgbe specific version of __pskb_pull_tail.  The
 * main difference between this version and the original function is that
 * this function can make several assumptions about the state of things
 * that allow for significant optimizations versus the standard function.
 * As a result we can do things like drop a frag and maintain an accurate
 * truesize for the skb.
 */
static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
                            struct sk_buff *skb)
{
        struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
        unsigned char *va;
        unsigned int pull_len;

        /*
         * it is valid to use page_address instead of kmap since we are
         * working with pages allocated out of the lomem pool per
         * alloc_page(GFP_ATOMIC)
         */
        va = skb_frag_address(frag);

        /*
         * we need the header to contain the greater of either ETH_HLEN or
         * 60 bytes if the skb->len is less than 60 for skb_pad.
         */
        pull_len = eth_get_headlen(va, IXGBE_RX_HDR_SIZE);

        /* align pull length to size of long to optimize memcpy performance */
        skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

        /* update all of the pointers */
        skb_frag_size_sub(frag, pull_len);
        frag->page_offset += pull_len;
        skb->data_len -= pull_len;
        skb->tail += pull_len;
}

/**
 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being updated
 *
 * This function provides a basic DMA sync up for the first fragment of an
 * skb.  The reason for doing this is that the first fragment cannot be
 * unmapped until we have reached the end of packet descriptor for a buffer
 * chain.
 */
static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
                                struct sk_buff *skb)
{
        /* if the page was released unmap it, else just sync our portion */
        if (unlikely(IXGBE_CB(skb)->page_released)) {
                dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma,
                                     ixgbe_rx_pg_size(rx_ring),
                                     DMA_FROM_DEVICE,
                                     IXGBE_RX_DMA_ATTR);
        } else if (ring_uses_build_skb(rx_ring)) {
                unsigned long offset = (unsigned long)(skb->data) & ~PAGE_MASK;

                dma_sync_single_range_for_cpu(rx_ring->dev,
                                              IXGBE_CB(skb)->dma,
                                              offset,
                                              skb_headlen(skb),
                                              DMA_FROM_DEVICE);
        } else {
                struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];

                dma_sync_single_range_for_cpu(rx_ring->dev,
                                              IXGBE_CB(skb)->dma,
                                              frag->page_offset,
                                              skb_frag_size(frag),
                                              DMA_FROM_DEVICE);
        }
}

/**
 * ixgbe_cleanup_headers - Correct corrupted or empty headers
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being fixed
 *
 * Check if the skb is valid in the XDP case it will be an error pointer.
 * Return true in this case to abort processing and advance to next
 * descriptor.
 *
 * Check for corrupted packet headers caused by senders on the local L2
 * embedded NIC switch not setting up their Tx Descriptors right.  These
 * should be very rare.
 *
 * Also address the case where we are pulling data in on pages only
 * and as such no data is present in the skb header.
 *
 * In addition if skb is not at least 60 bytes we need to pad it so that
 * it is large enough to qualify as a valid Ethernet frame.
 *
 * Returns true if an error was encountered and skb was freed.
 **/
static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
                                  union ixgbe_adv_rx_desc *rx_desc,
                                  struct sk_buff *skb)
{
        struct net_device *netdev = rx_ring->netdev;

        /* XDP packets use error pointer so abort at this point */
        if (IS_ERR(skb))
                return true;

        /* Verify netdev is present, and that packet does not have any
         * errors that would be unacceptable to the netdev.
         */
        if (!netdev ||
            (unlikely(ixgbe_test_staterr(rx_desc,
                                         IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
             !(netdev->features & NETIF_F_RXALL)))) {
                dev_kfree_skb_any(skb);
                return true;
        }

        /* place header in linear portion of buffer */
        if (!skb_headlen(skb))
                ixgbe_pull_tail(rx_ring, skb);

#ifdef IXGBE_FCOE
        /* do not attempt to pad FCoE Frames as this will disrupt DDP */
        if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
                return false;

#endif
        /* if eth_skb_pad returns an error the skb was freed */
        if (eth_skb_pad(skb))
                return true;

        return false;
}

/**
 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
 * @rx_ring: rx descriptor ring to store buffers on
 * @old_buff: donor buffer to have page reused
 *
 * Synchronizes page for reuse by the adapter
 **/
static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
                                struct ixgbe_rx_buffer *old_buff)
{
        struct ixgbe_rx_buffer *new_buff;
        u16 nta = rx_ring->next_to_alloc;

        new_buff = &rx_ring->rx_buffer_info[nta];

        /* update, and store next to alloc */
        nta++;
        rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

        /* Transfer page from old buffer to new buffer.
         * Move each member individually to avoid possible store
         * forwarding stalls and unnecessary copy of skb.
         */
        new_buff->dma           = old_buff->dma;
        new_buff->page          = old_buff->page;
        new_buff->page_offset   = old_buff->page_offset;
        new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
}

static inline bool ixgbe_page_is_reserved(struct page *page)
{

        return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
}

static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer)
{
        unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
        struct page *page = rx_buffer->page;

        /* avoid re-using remote pages */
        if (unlikely(ixgbe_page_is_reserved(page)))
                return false;

#if (PAGE_SIZE < 8192)
        /* if we are only owner of page we can reuse it */
        if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
                return false;
#else
        /* The last offset is a bit aggressive in that we assume the
         * worst case of FCoE being enabled and using a 3K buffer.
         * However this should have minimal impact as the 1K extra is
         * still less than one buffer in size.
         */
#define IXGBE_LAST_OFFSET \
        (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
        if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
                return false;
#endif

        /* If we have drained the page fragment pool we need to update
         * the pagecnt_bias and page count so that we fully restock the
         * number of references the driver holds.
         */
        if (unlikely(!pagecnt_bias)) {
                page_ref_add(page, USHRT_MAX);
                rx_buffer->pagecnt_bias = USHRT_MAX;
        }

        return true;
}

/**
 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_buffer: buffer containing page to add
 * @skb: sk_buff to place the data into
 * @size: size of data in rx_buffer
 *
 * This function will add the data contained in rx_buffer->page to the skb.
 * This is done either through a direct copy if the data in the buffer is
 * less than the skb header size, otherwise it will just attach the page as
 * a frag to the skb.
 *
 * The function will then update the page offset if necessary and return
 * true if the buffer can be reused by the adapter.
 **/
static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
                              struct ixgbe_rx_buffer *rx_buffer,
                              struct sk_buff *skb,
                              unsigned int size)
{
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = ring_uses_build_skb(rx_ring) ?
                                SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) :
                                SKB_DATA_ALIGN(size);
#endif
        skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
                        rx_buffer->page_offset, size, truesize);
#if (PAGE_SIZE < 8192)
        rx_buffer->page_offset ^= truesize;
#else
        rx_buffer->page_offset += truesize;
#endif
}

static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
                                                   union ixgbe_adv_rx_desc *rx_desc,
                                                   struct sk_buff **skb,
                                                   const unsigned int size)
{
        struct ixgbe_rx_buffer *rx_buffer;

        rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
        prefetchw(rx_buffer->page);
        *skb = rx_buffer->skb;

        /* Delay unmapping of the first packet. It carries the header
         * information, HW may still access the header after the writeback.
         * Only unmap it when EOP is reached
         */
        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
                if (!*skb)
                        goto skip_sync;
        } else {
                if (*skb)
                        ixgbe_dma_sync_frag(rx_ring, *skb);
        }

        /* we are reusing so sync this buffer for CPU use */
        dma_sync_single_range_for_cpu(rx_ring->dev,
                                      rx_buffer->dma,
                                      rx_buffer->page_offset,
                                      size,
                                      DMA_FROM_DEVICE);
skip_sync:
        rx_buffer->pagecnt_bias--;

        return rx_buffer;
}

static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
                                struct ixgbe_rx_buffer *rx_buffer,
                                struct sk_buff *skb)
{
        if (ixgbe_can_reuse_rx_page(rx_buffer)) {
                /* hand second half of page back to the ring */
                ixgbe_reuse_rx_page(rx_ring, rx_buffer);
        } else {
                if (!IS_ERR(skb) && IXGBE_CB(skb)->dma == rx_buffer->dma) {
                        /* the page has been released from the ring */
                        IXGBE_CB(skb)->page_released = true;
                } else {
                        /* we are not reusing the buffer so unmap it */
                        dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
                                             ixgbe_rx_pg_size(rx_ring),
                                             DMA_FROM_DEVICE,
                                             IXGBE_RX_DMA_ATTR);
                }
                __page_frag_cache_drain(rx_buffer->page,
                                        rx_buffer->pagecnt_bias);
        }

        /* clear contents of rx_buffer */
        rx_buffer->page = NULL;
        rx_buffer->skb = NULL;
}

static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
                                           struct ixgbe_rx_buffer *rx_buffer,
                                           struct xdp_buff *xdp,
                                           union ixgbe_adv_rx_desc *rx_desc)
{
        unsigned int size = xdp->data_end - xdp->data;
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
                                               xdp->data_hard_start);
#endif
        struct sk_buff *skb;

        /* prefetch first cache line of first page */
        prefetch(xdp->data);
#if L1_CACHE_BYTES < 128
        prefetch(xdp->data + L1_CACHE_BYTES);
#endif
        /* Note, we get here by enabling legacy-rx via:
         *
         *    ethtool --set-priv-flags <dev> legacy-rx on
         *
         * In this mode, we currently get 0 extra XDP headroom as
         * opposed to having legacy-rx off, where we process XDP
         * packets going to stack via ixgbe_build_skb(). The latter
         * provides us currently with 192 bytes of headroom.
         *
         * For ixgbe_construct_skb() mode it means that the
         * xdp->data_meta will always point to xdp->data, since
         * the helper cannot expand the head. Should this ever
         * change in future for legacy-rx mode on, then lets also
         * add xdp->data_meta handling here.
         */

        /* allocate a skb to store the frags */
        skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
        if (unlikely(!skb))
                return NULL;

        if (size > IXGBE_RX_HDR_SIZE) {
                if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
                        IXGBE_CB(skb)->dma = rx_buffer->dma;

                skb_add_rx_frag(skb, 0, rx_buffer->page,
                                xdp->data - page_address(rx_buffer->page),
                                size, truesize);
#if (PAGE_SIZE < 8192)
                rx_buffer->page_offset ^= truesize;
#else
                rx_buffer->page_offset += truesize;
#endif
        } else {
                memcpy(__skb_put(skb, size),
                       xdp->data, ALIGN(size, sizeof(long)));
                rx_buffer->pagecnt_bias++;
        }

        return skb;
}

static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
                                       struct ixgbe_rx_buffer *rx_buffer,
                                       struct xdp_buff *xdp,
                                       union ixgbe_adv_rx_desc *rx_desc)
{
        unsigned int metasize = xdp->data - xdp->data_meta;
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
                                SKB_DATA_ALIGN(xdp->data_end -
                                               xdp->data_hard_start);
#endif
        struct sk_buff *skb;

        /* Prefetch first cache line of first page. If xdp->data_meta
         * is unused, this points extactly as xdp->data, otherwise we
         * likely have a consumer accessing first few bytes of meta
         * data, and then actual data.
         */
        prefetch(xdp->data_meta);
#if L1_CACHE_BYTES < 128
        prefetch(xdp->data_meta + L1_CACHE_BYTES);
#endif

        /* build an skb to around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);
        if (unlikely(!skb))
                return NULL;

        /* update pointers within the skb to store the data */
        skb_reserve(skb, xdp->data - xdp->data_hard_start);
        __skb_put(skb, xdp->data_end - xdp->data);
        if (metasize)
                skb_metadata_set(skb, metasize);

        /* record DMA address if this is the start of a chain of buffers */
        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
                IXGBE_CB(skb)->dma = rx_buffer->dma;

        /* update buffer offset */
#if (PAGE_SIZE < 8192)
        rx_buffer->page_offset ^= truesize;
#else
        rx_buffer->page_offset += truesize;
#endif

        return skb;
}

#define IXGBE_XDP_PASS 0
#define IXGBE_XDP_CONSUMED 1
#define IXGBE_XDP_TX 2

static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
                               struct xdp_buff *xdp);

static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
                                     struct ixgbe_ring *rx_ring,
                                     struct xdp_buff *xdp)
{
        int err, result = IXGBE_XDP_PASS;
        struct bpf_prog *xdp_prog;
        u32 act;

        rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);

        if (!xdp_prog)
                goto xdp_out;

        act = bpf_prog_run_xdp(xdp_prog, xdp);
        switch (act) {
        case XDP_PASS:
                break;
        case XDP_TX:
                result = ixgbe_xmit_xdp_ring(adapter, xdp);
                break;
        case XDP_REDIRECT:
                err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
                if (!err)
                        result = IXGBE_XDP_TX;
                else
                        result = IXGBE_XDP_CONSUMED;
                break;
        default:
                bpf_warn_invalid_xdp_action(act);
                /* fallthrough */
        case XDP_ABORTED:
                trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
                /* fallthrough -- handle aborts by dropping packet */
        case XDP_DROP:
                result = IXGBE_XDP_CONSUMED;
                break;
        }
xdp_out:
        rcu_read_unlock();
        return ERR_PTR(-result);
}

static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring,
                                 struct ixgbe_rx_buffer *rx_buffer,
                                 unsigned int size)
{
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;

        rx_buffer->page_offset ^= truesize;
#else
        unsigned int truesize = ring_uses_build_skb(rx_ring) ?
                                SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) :
                                SKB_DATA_ALIGN(size);

        rx_buffer->page_offset += truesize;
#endif
}

/**
 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 * @q_vector: structure containing interrupt and ring information
 * @rx_ring: rx descriptor ring to transact packets on
 * @budget: Total limit on number of packets to process
 *
 * This function provides a "bounce buffer" approach to Rx interrupt
 * processing.  The advantage to this is that on systems that have
 * expensive overhead for IOMMU access this provides a means of avoiding
 * it by maintaining the mapping of the page to the syste.
 *
 * Returns amount of work completed
 **/
static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
                               struct ixgbe_ring *rx_ring,
                               const int budget)
{
        unsigned int total_rx_bytes = 0, total_rx_packets = 0;
        struct ixgbe_adapter *adapter = q_vector->adapter;
#ifdef IXGBE_FCOE
        int ddp_bytes;
        unsigned int mss = 0;
#endif /* IXGBE_FCOE */
        u16 cleaned_count = ixgbe_desc_unused(rx_ring);
        bool xdp_xmit = false;
        struct xdp_buff xdp;

        xdp.rxq = &rx_ring->xdp_rxq;

        while (likely(total_rx_packets < budget)) {
                union ixgbe_adv_rx_desc *rx_desc;
                struct ixgbe_rx_buffer *rx_buffer;
                struct sk_buff *skb;
                unsigned int size;

                /* return some buffers to hardware, one at a time is too slow */
                if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
                        ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
                        cleaned_count = 0;
                }

                rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
                size = le16_to_cpu(rx_desc->wb.upper.length);
                if (!size)
                        break;

                /* This memory barrier is needed to keep us from reading
                 * any other fields out of the rx_desc until we know the
                 * descriptor has been written back
                 */
                dma_rmb();

                rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size);

                /* retrieve a buffer from the ring */
                if (!skb) {
                        xdp.data = page_address(rx_buffer->page) +
                                   rx_buffer->page_offset;
                        xdp.data_meta = xdp.data;
                        xdp.data_hard_start = xdp.data -
                                              ixgbe_rx_offset(rx_ring);
                        xdp.data_end = xdp.data + size;

                        skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);
                }

                if (IS_ERR(skb)) {
                        if (PTR_ERR(skb) == -IXGBE_XDP_TX) {
                                xdp_xmit = true;
                                ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);
                        } else {
                                rx_buffer->pagecnt_bias++;
                        }
                        total_rx_packets++;
                        total_rx_bytes += size;
                } else if (skb) {
                        ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
                } else if (ring_uses_build_skb(rx_ring)) {
                        skb = ixgbe_build_skb(rx_ring, rx_buffer,
                                              &xdp, rx_desc);
                } else {
                        skb = ixgbe_construct_skb(rx_ring, rx_buffer,
                                                  &xdp, rx_desc);
                }

                /* exit if we failed to retrieve a buffer */
                if (!skb) {
                        rx_ring->rx_stats.alloc_rx_buff_failed++;
                        rx_buffer->pagecnt_bias++;
                        break;
                }

                ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb);
                cleaned_count++;

                /* place incomplete frames back on ring for completion */
                if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
                        continue;

                /* verify the packet layout is correct */
                if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
                        continue;

                /* probably a little skewed due to removing CRC */
                total_rx_bytes += skb->len;

                /* populate checksum, timestamp, VLAN, and protocol */
                ixgbe_process_skb_fields(rx_ring, rx_desc, skb);

#ifdef IXGBE_FCOE
                /* if ddp, not passing to ULD unless for FCP_RSP or error */
                if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
                        ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
                        /* include DDPed FCoE data */
                        if (ddp_bytes > 0) {
                                if (!mss) {
                                        mss = rx_ring->netdev->mtu -
                                                sizeof(struct fcoe_hdr) -
                                                sizeof(struct fc_frame_header) -
                                                sizeof(struct fcoe_crc_eof);
                                        if (mss > 512)
                                                mss &= ~511;
                                }
                                total_rx_bytes += ddp_bytes;
                                total_rx_packets += DIV_ROUND_UP(ddp_bytes,
                                                                 mss);
                        }
                        if (!ddp_bytes) {
                                dev_kfree_skb_any(skb);
                                continue;
                        }
                }

#endif /* IXGBE_FCOE */
                ixgbe_rx_skb(q_vector, skb);

                /* update budget accounting */
                total_rx_packets++;
        }

        if (xdp_xmit) {
                struct ixgbe_ring *ring = adapter->xdp_ring[smp_processor_id()];

                /* Force memory writes to complete before letting h/w
                 * know there are new descriptors to fetch.
                 */
                wmb();
                writel(ring->next_to_use, ring->tail);

                xdp_do_flush_map();
        }

        u64_stats_update_begin(&rx_ring->syncp);
        rx_ring->stats.packets += total_rx_packets;
        rx_ring->stats.bytes += total_rx_bytes;
        u64_stats_update_end(&rx_ring->syncp);
        q_vector->rx.total_packets += total_rx_packets;
        q_vector->rx.total_bytes += total_rx_bytes;

        return total_rx_packets;
}

/**
 * ixgbe_configure_msix - Configure MSI-X hardware
 * @adapter: board private structure
 *
 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
 * interrupts.
 **/
static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
{
        struct ixgbe_q_vector *q_vector;
        int v_idx;
        u32 mask;

        /* Populate MSIX to EITR Select */
        if (adapter->num_vfs > 32) {
                u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
        }

        /*
         * Populate the IVAR table and set the ITR values to the
         * corresponding register.
         */
        for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
                struct ixgbe_ring *ring;
                q_vector = adapter->q_vector[v_idx];

                ixgbe_for_each_ring(ring, q_vector->rx)
                        ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);

                ixgbe_for_each_ring(ring, q_vector->tx)
                        ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);

                ixgbe_write_eitr(q_vector);
        }

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
                               v_idx);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                ixgbe_set_ivar(adapter, -1, 1, v_idx);
                break;
        default:
                break;
        }
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);

        /* set up to autoclear timer, and the vectors */
        mask = IXGBE_EIMS_ENABLE_MASK;
        mask &= ~(IXGBE_EIMS_OTHER |
                  IXGBE_EIMS_MAILBOX |
                  IXGBE_EIMS_LSC);

        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
}

/**
 * ixgbe_update_itr - update the dynamic ITR value based on statistics
 * @q_vector: structure containing interrupt and ring information
 * @ring_container: structure containing ring performance data
 *
 *      Stores a new ITR value based on packets and byte
 *      counts during the last interrupt.  The advantage of per interrupt
 *      computation is faster updates and more accurate ITR for the current
 *      traffic pattern.  Constants in this function were computed
 *      based on theoretical maximum wire speed and thresholds were set based
 *      on testing data as well as attempting to minimize response time
 *      while increasing bulk throughput.
 **/
static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
                             struct ixgbe_ring_container *ring_container)
{
        unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS |
                           IXGBE_ITR_ADAPTIVE_LATENCY;
        unsigned int avg_wire_size, packets, bytes;
        unsigned long next_update = jiffies;

        /* If we don't have any rings just leave ourselves set for maximum
         * possible latency so we take ourselves out of the equation.
         */
        if (!ring_container->ring)
                return;

        /* If we didn't update within up to 1 - 2 jiffies we can assume
         * that either packets are coming in so slow there hasn't been
         * any work, or that there is so much work that NAPI is dealing
         * with interrupt moderation and we don't need to do anything.
         */
        if (time_after(next_update, ring_container->next_update))
                goto clear_counts;

        packets = ring_container->total_packets;

        /* We have no packets to actually measure against. This means
         * either one of the other queues on this vector is active or
         * we are a Tx queue doing TSO with too high of an interrupt rate.
         *
         * When this occurs just tick up our delay by the minimum value
         * and hope that this extra delay will prevent us from being called
         * without any work on our queue.
         */
        if (!packets) {
                itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
                if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
                itr += ring_container->itr & IXGBE_ITR_ADAPTIVE_LATENCY;
                goto clear_counts;
        }

        bytes = ring_container->total_bytes;

        /* If packets are less than 4 or bytes are less than 9000 assume
         * insufficient data to use bulk rate limiting approach. We are
         * likely latency driven.
         */
        if (packets < 4 && bytes < 9000) {
                itr = IXGBE_ITR_ADAPTIVE_LATENCY;
                goto adjust_by_size;
        }

        /* Between 4 and 48 we can assume that our current interrupt delay
         * is only slightly too low. As such we should increase it by a small
         * fixed amount.
         */
        if (packets < 48) {
                itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
                if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
                goto clear_counts;
        }

        /* Between 48 and 96 is our "goldilocks" zone where we are working
         * out "just right". Just report that our current ITR is good for us.
         */
        if (packets < 96) {
                itr = q_vector->itr >> 2;
                goto clear_counts;
        }

        /* If packet count is 96 or greater we are likely looking at a slight
         * overrun of the delay we want. Try halving our delay to see if that
         * will cut the number of packets in half per interrupt.
         */
        if (packets < 256) {
                itr = q_vector->itr >> 3;
                if (itr < IXGBE_ITR_ADAPTIVE_MIN_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MIN_USECS;
                goto clear_counts;
        }

        /* The paths below assume we are dealing with a bulk ITR since number
         * of packets is 256 or greater. We are just going to have to compute
         * a value and try to bring the count under control, though for smaller
         * packet sizes there isn't much we can do as NAPI polling will likely
         * be kicking in sooner rather than later.
         */
        itr = IXGBE_ITR_ADAPTIVE_BULK;

adjust_by_size:
        /* If packet counts are 256 or greater we can assume we have a gross
         * overestimation of what the rate should be. Instead of trying to fine
         * tune it just use the formula below to try and dial in an exact value
         * give the current packet size of the frame.
         */
        avg_wire_size = bytes / packets;

        /* The following is a crude approximation of:
         *  wmem_default / (size + overhead) = desired_pkts_per_int
         *  rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
         *  (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
         *
         * Assuming wmem_default is 212992 and overhead is 640 bytes per
         * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
         * formula down to
         *
         *  (170 * (size + 24)) / (size + 640) = ITR
         *
         * We first do some math on the packet size and then finally bitshift
         * by 8 after rounding up. We also have to account for PCIe link speed
         * difference as ITR scales based on this.
         */
        if (avg_wire_size <= 60) {
                /* Start at 50k ints/sec */
                avg_wire_size = 5120;
        } else if (avg_wire_size <= 316) {
                /* 50K ints/sec to 16K ints/sec */
                avg_wire_size *= 40;
                avg_wire_size += 2720;
        } else if (avg_wire_size <= 1084) {
                /* 16K ints/sec to 9.2K ints/sec */
                avg_wire_size *= 15;
                avg_wire_size += 11452;
        } else if (avg_wire_size <= 1980) {
                /* 9.2K ints/sec to 8K ints/sec */
                avg_wire_size *= 5;
                avg_wire_size += 22420;
        } else {
                /* plateau at a limit of 8K ints/sec */
                avg_wire_size = 32256;
        }

        /* If we are in low latency mode half our delay which doubles the rate
         * to somewhere between 100K to 16K ints/sec
         */
        if (itr & IXGBE_ITR_ADAPTIVE_LATENCY)
                avg_wire_size >>= 1;

        /* Resultant value is 256 times larger than it needs to be. This
         * gives us room to adjust the value as needed to either increase
         * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
         *
         * Use addition as we have already recorded the new latency flag
         * for the ITR value.
         */
        switch (q_vector->adapter->link_speed) {
        case IXGBE_LINK_SPEED_10GB_FULL:
        case IXGBE_LINK_SPEED_100_FULL:
        default:
                itr += DIV_ROUND_UP(avg_wire_size,
                                    IXGBE_ITR_ADAPTIVE_MIN_INC * 256) *
                       IXGBE_ITR_ADAPTIVE_MIN_INC;
                break;
        case IXGBE_LINK_SPEED_2_5GB_FULL:
        case IXGBE_LINK_SPEED_1GB_FULL:
        case IXGBE_LINK_SPEED_10_FULL:
                itr += DIV_ROUND_UP(avg_wire_size,
                                    IXGBE_ITR_ADAPTIVE_MIN_INC * 64) *
                       IXGBE_ITR_ADAPTIVE_MIN_INC;
                break;
        }

clear_counts:
        /* write back value */
        ring_container->itr = itr;

        /* next update should occur within next jiffy */
        ring_container->next_update = next_update + 1;

        ring_container->total_bytes = 0;
        ring_container->total_packets = 0;
}

/**
 * ixgbe_write_eitr - write EITR register in hardware specific way
 * @q_vector: structure containing interrupt and ring information
 *
 * This function is made to be called by ethtool and by the driver
 * when it needs to update EITR registers at runtime.  Hardware
 * specific quirks/differences are taken care of here.
 */
void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_hw *hw = &adapter->hw;
        int v_idx = q_vector->v_idx;
        u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                /* must write high and low 16 bits to reset counter */
                itr_reg |= (itr_reg << 16);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                /*
                 * set the WDIS bit to not clear the timer bits and cause an
                 * immediate assertion of the interrupt
                 */
                itr_reg |= IXGBE_EITR_CNT_WDIS;
                break;
        default:
                break;
        }
        IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
}

static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
{
        u32 new_itr;

        ixgbe_update_itr(q_vector, &q_vector->tx);
        ixgbe_update_itr(q_vector, &q_vector->rx);

        /* use the smallest value of new ITR delay calculations */
        new_itr = min(q_vector->rx.itr, q_vector->tx.itr);

        /* Clear latency flag if set, shift into correct position */
        new_itr &= ~IXGBE_ITR_ADAPTIVE_LATENCY;
        new_itr <<= 2;

        if (new_itr != q_vector->itr) {
                /* save the algorithm value here */
                q_vector->itr = new_itr;

                ixgbe_write_eitr(q_vector);
        }
}

/**
 * ixgbe_check_overtemp_subtask - check for over temperature
 * @adapter: pointer to adapter
 **/
static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr = adapter->interrupt_event;
        s32 rc;

        if (test_bit(__IXGBE_DOWN, &adapter->state))
                return;

        if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
                return;

        adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;

        switch (hw->device_id) {
        case IXGBE_DEV_ID_82599_T3_LOM:
                /*
                 * Since the warning interrupt is for both ports
                 * we don't have to check if:
                 *  - This interrupt wasn't for our port.
                 *  - We may have missed the interrupt so always have to
                 *    check if we  got a LSC
                 */
                if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
                    !(eicr & IXGBE_EICR_LSC))
                        return;

                if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
                        u32 speed;
                        bool link_up = false;

                        hw->mac.ops.check_link(hw, &speed, &link_up, false);

                        if (link_up)
                                return;
                }

                /* Check if this is not due to overtemp */
                if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
                        return;

                break;
        case IXGBE_DEV_ID_X550EM_A_1G_T:
        case IXGBE_DEV_ID_X550EM_A_1G_T_L:
                rc = hw->phy.ops.check_overtemp(hw);
                if (rc != IXGBE_ERR_OVERTEMP)
                        return;
                break;
        default:
                if (adapter->hw.mac.type >= ixgbe_mac_X540)
                        return;
                if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
                        return;
                break;
        }
        e_crit(drv, "%s\n", ixgbe_overheat_msg);

        adapter->interrupt_event = 0;
}

static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;

        if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
            (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
                e_crit(probe, "Fan has stopped, replace the adapter\n");
                /* write to clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
        }
}

static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;

        if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
                return;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
                /*
                 * Need to check link state so complete overtemp check
                 * on service task
                 */
                if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
                     (eicr & IXGBE_EICR_LSC)) &&
                    (!test_bit(__IXGBE_DOWN, &adapter->state))) {
                        adapter->interrupt_event = eicr;
                        adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
                        ixgbe_service_event_schedule(adapter);
                        return;
                }
                return;
        case ixgbe_mac_x550em_a:
                if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {
                        adapter->interrupt_event = eicr;
                        adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,
                                        IXGBE_EICR_GPI_SDP0_X550EM_a);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,
                                        IXGBE_EICR_GPI_SDP0_X550EM_a);
                }
                return;
        case ixgbe_mac_X550:
        case ixgbe_mac_X540:
                if (!(eicr & IXGBE_EICR_TS))
                        return;
                break;
        default:
                return;
        }

        e_crit(drv, "%s\n", ixgbe_overheat_msg);
}

static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
{
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                if (hw->phy.type == ixgbe_phy_nl)
                        return true;
                return false;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                switch (hw->mac.ops.get_media_type(hw)) {
                case ixgbe_media_type_fiber:
                case ixgbe_media_type_fiber_qsfp:
                        return true;
                default:
                        return false;
                }
        default:
                return false;
        }
}

static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);

        if (!ixgbe_is_sfp(hw))
                return;

        /* Later MAC's use different SDP */
        if (hw->mac.type >= ixgbe_mac_X540)
                eicr_mask = IXGBE_EICR_GPI_SDP0_X540;

        if (eicr & eicr_mask) {
                /* Clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
                if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                        adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
                        adapter->sfp_poll_time = 0;
                        ixgbe_service_event_schedule(adapter);
                }
        }

        if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
            (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
                /* Clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
                if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                        adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
                        ixgbe_service_event_schedule(adapter);
                }
        }
}

static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;

        adapter->lsc_int++;
        adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
        adapter->link_check_timeout = jiffies;
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
                IXGBE_WRITE_FLUSH(hw);
                ixgbe_service_event_schedule(adapter);
        }
}

static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
                                           u64 qmask)
{
        u32 mask;
        struct ixgbe_hw *hw = &adapter->hw;

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
                IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                mask = (qmask & 0xFFFFFFFF);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
                mask = (qmask >> 32);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
                break;
        default:
                break;

        }
        /* skip the flush */
}

static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
                                            u64 qmask)
{
        u32 mask;
        struct ixgbe_hw *hw = &adapter->hw;

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
                IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                mask = (qmask & 0xFFFFFFFF);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
                mask = (qmask >> 32);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
                break;
        default:
                break;
        }
        /* skip the flush */
}

/**
 * ixgbe_irq_enable - Enable default interrupt generation settings
 * @adapter: board private structure
 * @queues: enable irqs for queues
 * @flush: flush register write
 **/
static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
                                    bool flush)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);

        /* don't reenable LSC while waiting for link */
        if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
                mask &= ~IXGBE_EIMS_LSC;

        if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
                switch (adapter->hw.mac.type) {
                case ixgbe_mac_82599EB:
                        mask |= IXGBE_EIMS_GPI_SDP0(hw);
                        break;
                case ixgbe_mac_X540:
                case ixgbe_mac_X550:
                case ixgbe_mac_X550EM_x:
                case ixgbe_mac_x550em_a:
                        mask |= IXGBE_EIMS_TS;
                        break;
                default:
                        break;
                }
        if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
                mask |= IXGBE_EIMS_GPI_SDP1(hw);
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
                mask |= IXGBE_EIMS_GPI_SDP1(hw);
                mask |= IXGBE_EIMS_GPI_SDP2(hw);
                /* fall through */
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
                    adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
                    adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
                        mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
                if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
                        mask |= IXGBE_EICR_GPI_SDP0_X540;
                mask |= IXGBE_EIMS_ECC;
                mask |= IXGBE_EIMS_MAILBOX;
                break;
        default:
                break;
        }

        if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
            !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
                mask |= IXGBE_EIMS_FLOW_DIR;

        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
        if (queues)
                ixgbe_irq_enable_queues(adapter, ~0);
        if (flush)
                IXGBE_WRITE_FLUSH(&adapter->hw);
}

static irqreturn_t ixgbe_msix_other(int irq, void *data)
{
        struct ixgbe_adapter *adapter = data;
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr;

        /*
         * Workaround for Silicon errata.  Use clear-by-write instead
         * of clear-by-read.  Reading with EICS will return the
         * interrupt causes without clearing, which later be done
         * with the write to EICR.
         */
        eicr = IXGBE_READ_REG(hw, IXGBE_EICS);

        /* The lower 16bits of the EICR register are for the queue interrupts
         * which should be masked here in order to not accidentally clear them if
         * the bits are high when ixgbe_msix_other is called. There is a race
         * condition otherwise which results in possible performance loss
         * especially if the ixgbe_msix_other interrupt is triggering
         * consistently (as it would when PPS is turned on for the X540 device)
         */
        eicr &= 0xFFFF0000;

        IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);

        if (eicr & IXGBE_EICR_LSC)
                ixgbe_check_lsc(adapter);

        if (eicr & IXGBE_EICR_MAILBOX)
                ixgbe_msg_task(adapter);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
                    (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
                        adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR,
                                        IXGBE_EICR_GPI_SDP0_X540);
                }
                if (eicr & IXGBE_EICR_ECC) {
                        e_info(link, "Received ECC Err, initiating reset\n");
                        set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
                }
                /* Handle Flow Director Full threshold interrupt */
                if (eicr & IXGBE_EICR_FLOW_DIR) {
                        int reinit_count = 0;
                        int i;
                        for (i = 0; i < adapter->num_tx_queues; i++) {
                                struct ixgbe_ring *ring = adapter->tx_ring[i];
                                if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
                                                       &ring->state))
                                        reinit_count++;
                        }
                        if (reinit_count) {
                                /* no more flow director interrupts until after init */
                                IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
                                adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
                                ixgbe_service_event_schedule(adapter);
                        }
                }
                ixgbe_check_sfp_event(adapter, eicr);
                ixgbe_check_overtemp_event(adapter, eicr);
                break;
        default:
                break;
        }

        ixgbe_check_fan_failure(adapter, eicr);

        if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
                ixgbe_ptp_check_pps_event(adapter);

        /* re-enable the original interrupt state, no lsc, no queues */
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
                ixgbe_irq_enable(adapter, false, false);

        return IRQ_HANDLED;
}

static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
{
        struct ixgbe_q_vector *q_vector = data;

        /* EIAM disabled interrupts (on this vector) for us */

        if (q_vector->rx.ring || q_vector->tx.ring)
                napi_schedule_irqoff(&q_vector->napi);

        return IRQ_HANDLED;
}

/**
 * ixgbe_poll - NAPI Rx polling callback
 * @napi: structure for representing this polling device
 * @budget: how many packets driver is allowed to clean
 *
 * This function is used for legacy and MSI, NAPI mode
 **/
int ixgbe_poll(struct napi_struct *napi, int budget)
{
        struct ixgbe_q_vector *q_vector =
                                container_of(napi, struct ixgbe_q_vector, napi);
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_ring *ring;
        int per_ring_budget, work_done = 0;
        bool clean_complete = true;

#ifdef CONFIG_IXGBE_DCA
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                ixgbe_update_dca(q_vector);
#endif

        ixgbe_for_each_ring(ring, q_vector->tx) {
                if (!ixgbe_clean_tx_irq(q_vector, ring, budget))
                        clean_complete = false;
        }

        /* Exit if we are called by netpoll */
        if (budget <= 0)
                return budget;

        /* attempt to distribute budget to each queue fairly, but don't allow
         * the budget to go below 1 because we'll exit polling */
        if (q_vector->rx.count > 1)
                per_ring_budget = max(budget/q_vector->rx.count, 1);
        else
                per_ring_budget = budget;

        ixgbe_for_each_ring(ring, q_vector->rx) {
                int cleaned = ixgbe_clean_rx_irq(q_vector, ring,
                                                 per_ring_budget);

                work_done += cleaned;
                if (cleaned >= per_ring_budget)
                        clean_complete = false;
        }

        /* If all work not completed, return budget and keep polling */
        if (!clean_complete)
                return budget;

        /* all work done, exit the polling mode */
        napi_complete_done(napi, work_done);
        if (adapter->rx_itr_setting & 1)
                ixgbe_set_itr(q_vector);
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
                ixgbe_irq_enable_queues(adapter, BIT_ULL(q_vector->v_idx));

        return min(work_done, budget - 1);
}

/**
 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
 * @adapter: board private structure
 *
 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
 * interrupts from the kernel.
 **/
static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        unsigned int ri = 0, ti = 0;
        int vector, err;

        for (vector = 0; vector < adapter->num_q_vectors; vector++) {
                struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
                struct msix_entry *entry = &adapter->msix_entries[vector];

                if (q_vector->tx.ring && q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-TxRx-%u", netdev->name, ri++);
                        ti++;
                } else if (q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-rx-%u", netdev->name, ri++);
                } else if (q_vector->tx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-tx-%u", netdev->name, ti++);
                } else {
                        /* skip this unused q_vector */
                        continue;
                }
                err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
                                  q_vector->name, q_vector);
                if (err) {
                        e_err(probe, "request_irq failed for MSIX interrupt "
                              "Error: %d\n", err);
                        goto free_queue_irqs;
                }
                /* If Flow Director is enabled, set interrupt affinity */
                if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
                        /* assign the mask for this irq */
                        irq_set_affinity_hint(entry->vector,
                                              &q_vector->affinity_mask);
                }
        }

        err = request_irq(adapter->msix_entries[vector].vector,
                          ixgbe_msix_other, 0, netdev->name, adapter);
        if (err) {
                e_err(probe, "request_irq for msix_other failed: %d\n", err);
                goto free_queue_irqs;
        }

        return 0;

free_queue_irqs:
        while (vector) {
                vector--;
                irq_set_affinity_hint(adapter->msix_entries[vector].vector,
                                      NULL);
                free_irq(adapter->msix_entries[vector].vector,
                         adapter->q_vector[vector]);
        }
        adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
        pci_disable_msix(adapter->pdev);
        kfree(adapter->msix_entries);
        adapter->msix_entries = NULL;
        return err;
}

/**
 * ixgbe_intr - legacy mode Interrupt Handler
 * @irq: interrupt number
 * @data: pointer to a network interface device structure
 **/
static irqreturn_t ixgbe_intr(int irq, void *data)
{
        struct ixgbe_adapter *adapter = data;
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
        u32 eicr;

        /*
         * Workaround for silicon errata #26 on 82598.  Mask the interrupt
         * before the read of EICR.
         */
        IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);

        /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
         * therefore no explicit interrupt disable is necessary */
        eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
        if (!eicr) {
                /*
                 * shared interrupt alert!
                 * make sure interrupts are enabled because the read will
                 * have disabled interrupts due to EIAM
                 * finish the workaround of silicon errata on 82598.  Unmask
                 * the interrupt that we masked before the EICR read.
                 */
                if (!test_bit(__IXGBE_DOWN, &adapter->state))
                        ixgbe_irq_enable(adapter, true, true);
                return IRQ_NONE;        /* Not our interrupt */
        }

        if (eicr & IXGBE_EICR_LSC)
                ixgbe_check_lsc(adapter);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
                ixgbe_check_sfp_event(adapter, eicr);
                /* Fall through */
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (eicr & IXGBE_EICR_ECC) {
                        e_info(link, "Received ECC Err, initiating reset\n");
                        set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
                }
                ixgbe_check_overtemp_event(adapter, eicr);
                break;
        default:
                break;
        }

        ixgbe_check_fan_failure(adapter, eicr);
        if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
                ixgbe_ptp_check_pps_event(adapter);

        /* would disable interrupts here but EIAM disabled it */
        napi_schedule_irqoff(&q_vector->napi);

        /*
         * re-enable link(maybe) and non-queue interrupts, no flush.
         * ixgbe_poll will re-enable the queue interrupts
         */
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
                ixgbe_irq_enable(adapter, false, false);

        return IRQ_HANDLED;
}

/**
 * ixgbe_request_irq - initialize interrupts
 * @adapter: board private structure
 *
 * Attempts to configure interrupts using the best available
 * capabilities of the hardware and kernel.
 **/
static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        int err;

        if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
                err = ixgbe_request_msix_irqs(adapter);
        else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
                err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
                                  netdev->name, adapter);
        else
                err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
                                  netdev->name, adapter);

        if (err)
                e_err(probe, "request_irq failed, Error %d\n", err);

        return err;
}

static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
{
        int vector;

        if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
                free_irq(adapter->pdev->irq, adapter);
                return;
        }

        if (!adapter->msix_entries)
                return;

        for (vector = 0; vector < adapter->num_q_vectors; vector++) {
                struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
                struct msix_entry *entry = &adapter->msix_entries[vector];

                /* free only the irqs that were actually requested */
                if (!q_vector->rx.ring && !q_vector->tx.ring)
                        continue;

                /* clear the affinity_mask in the IRQ descriptor */
                irq_set_affinity_hint(entry->vector, NULL);

                free_irq(entry->vector, q_vector);
        }

        free_irq(adapter->msix_entries[vector].vector, adapter);
}

/**
 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
 * @adapter: board private structure
 **/
static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
{
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
                break;
        default:
                break;
        }
        IXGBE_WRITE_FLUSH(&adapter->hw);
        if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
                int vector;

                for (vector = 0; vector < adapter->num_q_vectors; vector++)
                        synchronize_irq(adapter->msix_entries[vector].vector);

                synchronize_irq(adapter->msix_entries[vector++].vector);
        } else {
                synchronize_irq(adapter->pdev->irq);
        }
}

/**
 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
 * @adapter: board private structure
 *
 **/
static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
{
        struct ixgbe_q_vector *q_vector = adapter->q_vector[0];

        ixgbe_write_eitr(q_vector);

        ixgbe_set_ivar(adapter, 0, 0, 0);
        ixgbe_set_ivar(adapter, 1, 0, 0);

        e_info(hw, "Legacy interrupt IVAR setup done\n");
}

/**
 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
 * @adapter: board private structure
 * @ring: structure containing ring specific data
 *
 * Configure the Tx descriptor ring after a reset.
 **/
void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
                             struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u64 tdba = ring->dma;
        int wait_loop = 10;
        u32 txdctl = IXGBE_TXDCTL_ENABLE;
        u8 reg_idx = ring->reg_idx;

        /* disable queue to avoid issues while updating state */
        IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
        IXGBE_WRITE_FLUSH(hw);

        IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
                        (tdba & DMA_BIT_MASK(32)));
        IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
        IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
                        ring->count * sizeof(union ixgbe_adv_tx_desc));
        IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
        IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
        ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);

        /*
         * set WTHRESH to encourage burst writeback, it should not be set
         * higher than 1 when:
         * - ITR is 0 as it could cause false TX hangs
         * - ITR is set to > 100k int/sec and BQL is enabled
         *
         * In order to avoid issues WTHRESH + PTHRESH should always be equal
         * to or less than the number of on chip descriptors, which is
         * currently 40.
         */
        if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
                txdctl |= 1u << 16;     /* WTHRESH = 1 */
        else
                txdctl |= 8u << 16;     /* WTHRESH = 8 */

        /*
         * Setting PTHRESH to 32 both improves performance
         * and avoids a TX hang with DFP enabled
         */
        txdctl |= (1u << 8) |   /* HTHRESH = 1 */
                   32;          /* PTHRESH = 32 */

        /* reinitialize flowdirector state */
        if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
                ring->atr_sample_rate = adapter->atr_sample_rate;
                ring->atr_count = 0;
                set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
        } else {
                ring->atr_sample_rate = 0;
        }

        /* initialize XPS */
        if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {
                struct ixgbe_q_vector *q_vector = ring->q_vector;

                if (q_vector)
                        netif_set_xps_queue(ring->netdev,
                                            &q_vector->affinity_mask,
                                            ring->queue_index);
        }

        clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);

        /* reinitialize tx_buffer_info */
        memset(ring->tx_buffer_info, 0,
               sizeof(struct ixgbe_tx_buffer) * ring->count);

        /* enable queue */
        IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);

        /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
        if (hw->mac.type == ixgbe_mac_82598EB &&
            !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
                return;

        /* poll to verify queue is enabled */
        do {
                usleep_range(1000, 2000);
                txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
        } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
        if (!wait_loop)
                hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
}

static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 rttdcs, mtqc;
        u8 tcs = adapter->hw_tcs;

        if (hw->mac.type == ixgbe_mac_82598EB)
                return;

        /* disable the arbiter while setting MTQC */
        rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
        rttdcs |= IXGBE_RTTDCS_ARBDIS;
        IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);

        /* set transmit pool layout */
        if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
                mtqc = IXGBE_MTQC_VT_ENA;
                if (tcs > 4)
                        mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
                else if (tcs > 1)
                        mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
                else if (adapter->ring_feature[RING_F_VMDQ].mask ==
                         IXGBE_82599_VMDQ_4Q_MASK)
                        mtqc |= IXGBE_MTQC_32VF;
                else
                        mtqc |= IXGBE_MTQC_64VF;
        } else {
                if (tcs > 4)
                        mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
                else if (tcs > 1)
                        mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
                else
                        mtqc = IXGBE_MTQC_64Q_1PB;
        }

        IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);

        /* Enable Security TX Buffer IFG for multiple pb */
        if (tcs) {
                u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
                sectx |= IXGBE_SECTX_DCB;
                IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
        }

        /* re-enable the arbiter */
        rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
        IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
}

/**
 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
 * @adapter: board private structure
 *
 * Configure the Tx unit of the MAC after a reset.
 **/
static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 dmatxctl;
        u32 i;

        ixgbe_setup_mtqc(adapter);

        if (hw->mac.type != ixgbe_mac_82598EB) {
                /* DMATXCTL.EN must be before Tx queues are enabled */
                dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
                dmatxctl |= IXGBE_DMATXCTL_TE;
                IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
        }

        /* Setup the HW Tx Head and Tail descriptor pointers */
        for (i = 0; i < adapter->num_tx_queues; i++)
                ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
        for (i = 0; i < adapter->num_xdp_queues; i++)
                ixgbe_configure_tx_ring(adapter, adapter->xdp_ring[i]);
}

static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
                                 struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u8 reg_idx = ring->reg_idx;
        u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

        srrctl |= IXGBE_SRRCTL_DROP_EN;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
                                  struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u8 reg_idx = ring->reg_idx;
        u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

        srrctl &= ~IXGBE_SRRCTL_DROP_EN;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

#ifdef CONFIG_IXGBE_DCB
void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#else
static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#endif
{
        int i;
        bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

        if (adapter->ixgbe_ieee_pfc)
                pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

        /*
         * We should set the drop enable bit if:
         *  SR-IOV is enabled
         *   or
         *  Number of Rx queues > 1 and flow control is disabled
         *
         *  This allows us to avoid head of line blocking for security
         *  and performance reasons.
         */
        if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
            !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
                for (i = 0; i < adapter->num_rx_queues; i++)
                        ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
        } else {
                for (i = 0; i < adapter->num_rx_queues; i++)
                        ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
        }
}

#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2

static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
                                   struct ixgbe_ring *rx_ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 srrctl;
        u8 reg_idx = rx_ring->reg_idx;

        if (hw->mac.type == ixgbe_mac_82598EB) {
                u16 mask = adapter->ring_feature[RING_F_RSS].mask;

                /*
                 * if VMDq is not active we must program one srrctl register
                 * per RSS queue since we have enabled RDRXCTL.MVMEN
                 */
                reg_idx &= mask;
        }

        /* configure header buffer length, needed for RSC */
        srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;

        /* configure the packet buffer length */
        if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state))
                srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
        else
                srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

        /* configure descriptor type */
        srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

/**
 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
 * @adapter: device handle
 *
 *  - 82598/82599/X540:     128
 *  - X550(non-SRIOV mode): 512
 *  - X550(SRIOV mode):     64
 */
u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
{
        if (adapter->hw.mac.type < ixgbe_mac_X550)
                return 128;
        else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
                return 64;
        else
                return 512;
}

/**
 * ixgbe_store_key - Write the RSS key to HW
 * @adapter: device handle
 *
 * Write the RSS key stored in adapter.rss_key to HW.
 */
void ixgbe_store_key(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        int i;

        for (i = 0; i < 10; i++)
                IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
}

/**
 * ixgbe_init_rss_key - Initialize adapter RSS key
 * @adapter: device handle
 *
 * Allocates and initializes the RSS key if it is not allocated.
 **/
static inline int ixgbe_init_rss_key(struct ixgbe_adapter *adapter)
{
        u32 *rss_key;

        if (!adapter->rss_key) {
                rss_key = kzalloc(IXGBE_RSS_KEY_SIZE, GFP_KERNEL);
                if (unlikely(!rss_key))
                        return -ENOMEM;

                netdev_rss_key_fill(rss_key, IXGBE_RSS_KEY_SIZE);
                adapter->rss_key = rss_key;
        }

        return 0;
}

/**
 * ixgbe_store_reta - Write the RETA table to HW
 * @adapter: device handle
 *
 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
 */
void ixgbe_store_reta(struct ixgbe_adapter *adapter)
{
        u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 reta = 0;
        u32 indices_multi;
        u8 *indir_tbl = adapter->rss_indir_tbl;

        /* Fill out the redirection table as follows:
         *  - 82598:      8 bit wide entries containing pair of 4 bit RSS
         *    indices.
         *  - 82599/X540: 8 bit wide entries containing 4 bit RSS index
         *  - X550:       8 bit wide entries containing 6 bit RSS index
         */
        if (adapter->hw.mac.type == ixgbe_mac_82598EB)
                indices_multi = 0x11;
        else
                indices_multi = 0x1;

        /* Write redirection table to HW */
        for (i = 0; i < reta_entries; i++) {
                reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
                if ((i & 3) == 3) {
                        if (i < 128)
                                IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
                        else
                                IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
                                                reta);
                        reta = 0;
                }
        }
}

/**
 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
 * @adapter: device handle
 *
 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
 */
static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
{
        u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 vfreta = 0;

        /* Write redirection table to HW */
        for (i = 0; i < reta_entries; i++) {
                u16 pool = adapter->num_rx_pools;

                vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
                if ((i & 3) != 3)
                        continue;

                while (pool--)
                        IXGBE_WRITE_REG(hw,
                                        IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),
                                        vfreta);
                vfreta = 0;
        }
}

static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
{
        u32 i, j;
        u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;

        /* Program table for at least 4 queues w/ SR-IOV so that VFs can
         * make full use of any rings they may have.  We will use the
         * PSRTYPE register to control how many rings we use within the PF.
         */
        if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
                rss_i = 4;

        /* Fill out hash function seeds */
        ixgbe_store_key(adapter);

        /* Fill out redirection table */
        memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));

        for (i = 0, j = 0; i < reta_entries; i++, j++) {
                if (j == rss_i)
                        j = 0;

                adapter->rss_indir_tbl[i] = j;
        }

        ixgbe_store_reta(adapter);
}

static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
        int i, j;

        /* Fill out hash function seeds */
        for (i = 0; i < 10; i++) {
                u16 pool = adapter->num_rx_pools;

                while (pool--)
                        IXGBE_WRITE_REG(hw,
                                        IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),
                                        *(adapter->rss_key + i));
        }

        /* Fill out the redirection table */
        for (i = 0, j = 0; i < 64; i++, j++) {
                if (j == rss_i)
                        j = 0;

                adapter->rss_indir_tbl[i] = j;
        }

        ixgbe_store_vfreta(adapter);
}

static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
        u32 rxcsum;

        /* Disable indicating checksum in descriptor, enables RSS hash */
        rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
        rxcsum |= IXGBE_RXCSUM_PCSD;
        IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);

        if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
                if (adapter->ring_feature[RING_F_RSS].mask)
                        mrqc = IXGBE_MRQC_RSSEN;
        } else {
                u8 tcs = adapter->hw_tcs;

                if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
                        if (tcs > 4)
                                mrqc = IXGBE_MRQC_VMDQRT8TCEN;  /* 8 TCs */
                        else if (tcs > 1)
                                mrqc = IXGBE_MRQC_VMDQRT4TCEN;  /* 4 TCs */
                        else if (adapter->ring_feature[RING_F_VMDQ].mask ==
                                 IXGBE_82599_VMDQ_4Q_MASK)
                                mrqc = IXGBE_MRQC_VMDQRSS32EN;
                        else
                                mrqc = IXGBE_MRQC_VMDQRSS64EN;

                        /* Enable L3/L4 for Tx Switched packets */
                        mrqc |= IXGBE_MRQC_L3L4TXSWEN;
                } else {
                        if (tcs > 4)
                                mrqc = IXGBE_MRQC_RTRSS8TCEN;
                        else if (tcs > 1)
                                mrqc = IXGBE_MRQC_RTRSS4TCEN;
                        else
                                mrqc = IXGBE_MRQC_RSSEN;
                }
        }

        /* Perform hash on these packet types */
        rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
                     IXGBE_MRQC_RSS_FIELD_IPV6 |
                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP;

        if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
                rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
        if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
                rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;

        if ((hw->mac.type >= ixgbe_mac_X550) &&
            (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
                u16 pool = adapter->num_rx_pools;