// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 1999 - 2018 Intel Corporation. */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/interrupt.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
#include <linux/pkt_sched.h>
#include <linux/ipv6.h>
#include <linux/slab.h>
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/if.h>
#include <linux/if_vlan.h>
#include <linux/if_macvlan.h>
#include <linux/if_bridge.h>
#include <linux/prefetch.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/atomic.h>
#include <linux/numa.h>
#include <generated/utsrelease.h>
#include <scsi/fc/fc_fcoe.h>
#include <net/udp_tunnel.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_mirred.h>
#include <net/vxlan.h>
#include <net/mpls.h>
#include <net/xdp_sock_drv.h>
#include <net/xfrm.h>

#include "ixgbe.h"
#include "ixgbe_common.h"
#include "ixgbe_dcb_82599.h"
#include "ixgbe_phy.h"
#include "ixgbe_sriov.h"
#include "ixgbe_model.h"
#include "ixgbe_txrx_common.h"

char ixgbe_driver_name[] = "ixgbe";
static const char ixgbe_driver_string[] =
                              "Intel(R) 10 Gigabit PCI Express Network Driver";
#ifdef IXGBE_FCOE
char ixgbe_default_device_descr[] =
                              "Intel(R) 10 Gigabit Network Connection";
#else
static char ixgbe_default_device_descr[] =
                              "Intel(R) 10 Gigabit Network Connection";
#endif
static const char ixgbe_copyright[] =
                                "Copyright (c) 1999-2016 Intel Corporation.";

static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";

static const struct ixgbe_info *ixgbe_info_tbl[] = {
        [board_82598]           = &ixgbe_82598_info,
        [board_82599]           = &ixgbe_82599_info,
        [board_X540]            = &ixgbe_X540_info,
        [board_X550]            = &ixgbe_X550_info,
        [board_X550EM_x]        = &ixgbe_X550EM_x_info,
        [board_x550em_x_fw]     = &ixgbe_x550em_x_fw_info,
        [board_x550em_a]        = &ixgbe_x550em_a_info,
        [board_x550em_a_fw]     = &ixgbe_x550em_a_fw_info,
};

/* ixgbe_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
static const struct pci_device_id ixgbe_pci_tbl[] = {
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
        {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
        /* required last entry */
        {0, }
};
MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);

#ifdef CONFIG_IXGBE_DCA
static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
                            void *p);
static struct notifier_block dca_notifier = {
        .notifier_call = ixgbe_notify_dca,
        .next          = NULL,
        .priority      = 0
};
#endif

#ifdef CONFIG_PCI_IOV
static unsigned int max_vfs;
module_param(max_vfs, uint, 0);
MODULE_PARM_DESC(max_vfs,
                 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
#endif /* CONFIG_PCI_IOV */

static unsigned int allow_unsupported_sfp;
module_param(allow_unsupported_sfp, uint, 0);
MODULE_PARM_DESC(allow_unsupported_sfp,
                 "Allow unsupported and untested SFP+ modules on 82599-based adapters");

#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
MODULE_LICENSE("GPL v2");

static struct workqueue_struct *ixgbe_wq;

static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);

static const struct net_device_ops ixgbe_netdev_ops;

static bool netif_is_ixgbe(struct net_device *dev)
{
        return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
}

static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
                                          u32 reg, u16 *value)
{
        struct pci_dev *parent_dev;
        struct pci_bus *parent_bus;

        parent_bus = adapter->pdev->bus->parent;
        if (!parent_bus)
                return -1;

        parent_dev = parent_bus->self;
        if (!parent_dev)
                return -1;

        if (!pci_is_pcie(parent_dev))
                return -1;

        pcie_capability_read_word(parent_dev, reg, value);
        if (*value == IXGBE_FAILED_READ_CFG_WORD &&
            ixgbe_check_cfg_remove(&adapter->hw, parent_dev))
                return -1;
        return 0;
}

static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u16 link_status = 0;
        int err;

        hw->bus.type = ixgbe_bus_type_pci_express;

        /* Get the negotiated link width and speed from PCI config space of the
         * parent, as this device is behind a switch
         */
        err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);

        /* assume caller will handle error case */
        if (err)
                return err;

        hw->bus.width = ixgbe_convert_bus_width(link_status);
        hw->bus.speed = ixgbe_convert_bus_speed(link_status);

        return 0;
}

/**
 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent
 * @hw: hw specific details
 *
 * This function is used by probe to determine whether a device's PCI-Express
 * bandwidth details should be gathered from the parent bus instead of from the
 * device. Used to ensure that various locations all have the correct device ID
 * checks.
 */
static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
{
        switch (hw->device_id) {
        case IXGBE_DEV_ID_82599_SFP_SF_QP:
        case IXGBE_DEV_ID_82599_QSFP_SF_QP:
                return true;
        default:
                return false;
        }
}

static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
                                     int expected_gts)
{
        struct ixgbe_hw *hw = &adapter->hw;
        struct pci_dev *pdev;

        /* Some devices are not connected over PCIe and thus do not negotiate
         * speed. These devices do not have valid bus info, and thus any report
         * we generate may not be correct.
         */
        if (hw->bus.type == ixgbe_bus_type_internal)
                return;

        /* determine whether to use the parent device */
        if (ixgbe_pcie_from_parent(&adapter->hw))
                pdev = adapter->pdev->bus->parent->self;
        else
                pdev = adapter->pdev;

        pcie_print_link_status(pdev);
}

static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
{
        if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
            !test_bit(__IXGBE_REMOVING, &adapter->state) &&
            !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
                queue_work(ixgbe_wq, &adapter->service_task);
}

static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
{
        struct ixgbe_adapter *adapter = hw->back;

        if (!hw->hw_addr)
                return;
        hw->hw_addr = NULL;
        e_dev_err("Adapter removed\n");
        if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
                ixgbe_service_event_schedule(adapter);
}

static u32 ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
{
        u8 __iomem *reg_addr;
        u32 value;
        int i;

        reg_addr = READ_ONCE(hw->hw_addr);
        if (ixgbe_removed(reg_addr))
                return IXGBE_FAILED_READ_REG;

        /* Register read of 0xFFFFFFF can indicate the adapter has been removed,
         * so perform several status register reads to determine if the adapter
         * has been removed.
         */
        for (i = 0; i < IXGBE_FAILED_READ_RETRIES; i++) {
                value = readl(reg_addr + IXGBE_STATUS);
                if (value != IXGBE_FAILED_READ_REG)
                        break;
                mdelay(3);
        }

        if (value == IXGBE_FAILED_READ_REG)
                ixgbe_remove_adapter(hw);
        else
                value = readl(reg_addr + reg);
        return value;
}

/**
 * ixgbe_read_reg - Read from device register
 * @hw: hw specific details
 * @reg: offset of register to read
 *
 * Returns : value read or IXGBE_FAILED_READ_REG if removed
 *
 * This function is used to read device registers. It checks for device
 * removal by confirming any read that returns all ones by checking the
 * status register value for all ones. This function avoids reading from
 * the hardware if a removal was previously detected in which case it
 * returns IXGBE_FAILED_READ_REG (all ones).
 */
u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
{
        u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
        u32 value;

        if (ixgbe_removed(reg_addr))
                return IXGBE_FAILED_READ_REG;
        if (unlikely(hw->phy.nw_mng_if_sel &
                     IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
                struct ixgbe_adapter *adapter;
                int i;

                for (i = 0; i < 200; ++i) {
                        value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY);
                        if (likely(!value))
                                goto writes_completed;
                        if (value == IXGBE_FAILED_READ_REG) {
                                ixgbe_remove_adapter(hw);
                                return IXGBE_FAILED_READ_REG;
                        }
                        udelay(5);
                }

                adapter = hw->back;
                e_warn(hw, "register writes incomplete %08x\n", value);
        }

writes_completed:
        value = readl(reg_addr + reg);
        if (unlikely(value == IXGBE_FAILED_READ_REG))
                value = ixgbe_check_remove(hw, reg);
        return value;
}

static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
{
        u16 value;

        pci_read_config_word(pdev, PCI_VENDOR_ID, &value);
        if (value == IXGBE_FAILED_READ_CFG_WORD) {
                ixgbe_remove_adapter(hw);
                return true;
        }
        return false;
}

u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
{
        struct ixgbe_adapter *adapter = hw->back;
        u16 value;

        if (ixgbe_removed(hw->hw_addr))
                return IXGBE_FAILED_READ_CFG_WORD;
        pci_read_config_word(adapter->pdev, reg, &value);
        if (value == IXGBE_FAILED_READ_CFG_WORD &&
            ixgbe_check_cfg_remove(hw, adapter->pdev))
                return IXGBE_FAILED_READ_CFG_WORD;
        return value;
}

#ifdef CONFIG_PCI_IOV
static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
{
        struct ixgbe_adapter *adapter = hw->back;
        u32 value;

        if (ixgbe_removed(hw->hw_addr))
                return IXGBE_FAILED_READ_CFG_DWORD;
        pci_read_config_dword(adapter->pdev, reg, &value);
        if (value == IXGBE_FAILED_READ_CFG_DWORD &&
            ixgbe_check_cfg_remove(hw, adapter->pdev))
                return IXGBE_FAILED_READ_CFG_DWORD;
        return value;
}
#endif /* CONFIG_PCI_IOV */

void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
{
        struct ixgbe_adapter *adapter = hw->back;

        if (ixgbe_removed(hw->hw_addr))
                return;
        pci_write_config_word(adapter->pdev, reg, value);
}

static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
{
        BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));

        /* flush memory to make sure state is correct before next watchdog */
        smp_mb__before_atomic();
        clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
}

struct ixgbe_reg_info {
        u32 ofs;
        char *name;
};

static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {

        /* General Registers */
        {IXGBE_CTRL, "CTRL"},
        {IXGBE_STATUS, "STATUS"},
        {IXGBE_CTRL_EXT, "CTRL_EXT"},

        /* Interrupt Registers */
        {IXGBE_EICR, "EICR"},

        /* RX Registers */
        {IXGBE_SRRCTL(0), "SRRCTL"},
        {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
        {IXGBE_RDLEN(0), "RDLEN"},
        {IXGBE_RDH(0), "RDH"},
        {IXGBE_RDT(0), "RDT"},
        {IXGBE_RXDCTL(0), "RXDCTL"},
        {IXGBE_RDBAL(0), "RDBAL"},
        {IXGBE_RDBAH(0), "RDBAH"},

        /* TX Registers */
        {IXGBE_TDBAL(0), "TDBAL"},
        {IXGBE_TDBAH(0), "TDBAH"},
        {IXGBE_TDLEN(0), "TDLEN"},
        {IXGBE_TDH(0), "TDH"},
        {IXGBE_TDT(0), "TDT"},
        {IXGBE_TXDCTL(0), "TXDCTL"},

        /* List Terminator */
        { .name = NULL }
};


/*
 * ixgbe_regdump - register printout routine
 */
static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
{
        int i;
        char rname[16];
        u32 regs[64];

        switch (reginfo->ofs) {
        case IXGBE_SRRCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
                break;
        case IXGBE_DCA_RXCTRL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
                break;
        case IXGBE_RDLEN(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
                break;
        case IXGBE_RDH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
                break;
        case IXGBE_RDT(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
                break;
        case IXGBE_RXDCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
                break;
        case IXGBE_RDBAL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
                break;
        case IXGBE_RDBAH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
                break;
        case IXGBE_TDBAL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
                break;
        case IXGBE_TDBAH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
                break;
        case IXGBE_TDLEN(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
                break;
        case IXGBE_TDH(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
                break;
        case IXGBE_TDT(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
                break;
        case IXGBE_TXDCTL(0):
                for (i = 0; i < 64; i++)
                        regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
                break;
        default:
                pr_info("%-15s %08x\n",
                        reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));
                return;
        }

        i = 0;
        while (i < 64) {
                int j;
                char buf[9 * 8 + 1];
                char *p = buf;

                snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i, i + 7);
                for (j = 0; j < 8; j++)
                        p += sprintf(p, " %08x", regs[i++]);
                pr_err("%-15s%s\n", rname, buf);
        }

}

static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)
{
        struct ixgbe_tx_buffer *tx_buffer;

        tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];
        pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
                n, ring->next_to_use, ring->next_to_clean,
                (u64)dma_unmap_addr(tx_buffer, dma),
                dma_unmap_len(tx_buffer, len),
                tx_buffer->next_to_watch,
                (u64)tx_buffer->time_stamp);
}

/*
 * ixgbe_dump - Print registers, tx-rings and rx-rings
 */
static void ixgbe_dump(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_reg_info *reginfo;
        int n = 0;
        struct ixgbe_ring *ring;
        struct ixgbe_tx_buffer *tx_buffer;
        union ixgbe_adv_tx_desc *tx_desc;
        struct my_u0 { u64 a; u64 b; } *u0;
        struct ixgbe_ring *rx_ring;
        union ixgbe_adv_rx_desc *rx_desc;
        struct ixgbe_rx_buffer *rx_buffer_info;
        int i = 0;

        if (!netif_msg_hw(adapter))
                return;

        /* Print netdevice Info */
        if (netdev) {
                dev_info(&adapter->pdev->dev, "Net device Info\n");
                pr_info("Device Name     state            "
                        "trans_start\n");
                pr_info("%-15s %016lX %016lX\n",
                        netdev->name,
                        netdev->state,
                        dev_trans_start(netdev));
        }

        /* Print Registers */
        dev_info(&adapter->pdev->dev, "Register Dump\n");
        pr_info(" Register Name   Value\n");
        for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
             reginfo->name; reginfo++) {
                ixgbe_regdump(hw, reginfo);
        }

        /* Print TX Ring Summary */
        if (!netdev || !netif_running(netdev))
                return;

        dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
        pr_info(" %s     %s              %s        %s\n",
                "Queue [NTU] [NTC] [bi(ntc)->dma  ]",
                "leng", "ntw", "timestamp");
        for (n = 0; n < adapter->num_tx_queues; n++) {
                ring = adapter->tx_ring[n];
                ixgbe_print_buffer(ring, n);
        }

        for (n = 0; n < adapter->num_xdp_queues; n++) {
                ring = adapter->xdp_ring[n];
                ixgbe_print_buffer(ring, n);
        }

        /* Print TX Rings */
        if (!netif_msg_tx_done(adapter))
                goto rx_ring_summary;

        dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

        /* Transmit Descriptor Formats
         *
         * 82598 Advanced Transmit Descriptor
         *   +--------------------------------------------------------------+
         * 0 |         Buffer Address [63:0]                                |
         *   +--------------------------------------------------------------+
         * 8 |  PAYLEN  | POPTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
         *   +--------------------------------------------------------------+
         *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
         *
         * 82598 Advanced Transmit Descriptor (Write-Back Format)
         *   +--------------------------------------------------------------+
         * 0 |                          RSV [63:0]                          |
         *   +--------------------------------------------------------------+
         * 8 |            RSV           |  STA  |          NXTSEQ           |
         *   +--------------------------------------------------------------+
         *   63                       36 35   32 31                         0
         *
         * 82599+ Advanced Transmit Descriptor
         *   +--------------------------------------------------------------+
         * 0 |         Buffer Address [63:0]                                |
         *   +--------------------------------------------------------------+
         * 8 |PAYLEN  |POPTS|CC|IDX  |STA  |DCMD  |DTYP |MAC  |RSV  |DTALEN |
         *   +--------------------------------------------------------------+
         *   63     46 45 40 39 38 36 35 32 31  24 23 20 19 18 17 16 15     0
         *
         * 82599+ Advanced Transmit Descriptor (Write-Back Format)
         *   +--------------------------------------------------------------+
         * 0 |                          RSV [63:0]                          |
         *   +--------------------------------------------------------------+
         * 8 |            RSV           |  STA  |           RSV             |
         *   +--------------------------------------------------------------+
         *   63                       36 35   32 31                         0
         */

        for (n = 0; n < adapter->num_tx_queues; n++) {
                ring = adapter->tx_ring[n];
                pr_info("------------------------------------\n");
                pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);
                pr_info("------------------------------------\n");
                pr_info("%s%s    %s              %s        %s          %s\n",
                        "T [desc]     [address 63:0  ] ",
                        "[PlPOIdStDDt Ln] [bi->dma       ] ",
                        "leng", "ntw", "timestamp", "bi->skb");

                for (i = 0; ring->desc && (i < ring->count); i++) {
                        tx_desc = IXGBE_TX_DESC(ring, i);
                        tx_buffer = &ring->tx_buffer_info[i];
                        u0 = (struct my_u0 *)tx_desc;
                        if (dma_unmap_len(tx_buffer, len) > 0) {
                                const char *ring_desc;

                                if (i == ring->next_to_use &&
                                    i == ring->next_to_clean)
                                        ring_desc = " NTC/U";
                                else if (i == ring->next_to_use)
                                        ring_desc = " NTU";
                                else if (i == ring->next_to_clean)
                                        ring_desc = " NTC";
                                else
                                        ring_desc = "";
                                pr_info("T [0x%03X]    %016llX %016llX %016llX %08X %p %016llX %p%s",
                                        i,
                                        le64_to_cpu((__force __le64)u0->a),
                                        le64_to_cpu((__force __le64)u0->b),
                                        (u64)dma_unmap_addr(tx_buffer, dma),
                                        dma_unmap_len(tx_buffer, len),
                                        tx_buffer->next_to_watch,
                                        (u64)tx_buffer->time_stamp,
                                        tx_buffer->skb,
                                        ring_desc);

                                if (netif_msg_pktdata(adapter) &&
                                    tx_buffer->skb)
                                        print_hex_dump(KERN_INFO, "",
                                                DUMP_PREFIX_ADDRESS, 16, 1,
                                                tx_buffer->skb->data,
                                                dma_unmap_len(tx_buffer, len),
                                                true);
                        }
                }
        }

        /* Print RX Rings Summary */
rx_ring_summary:
        dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
        pr_info("Queue [NTU] [NTC]\n");
        for (n = 0; n < adapter->num_rx_queues; n++) {
                rx_ring = adapter->rx_ring[n];
                pr_info("%5d %5X %5X\n",
                        n, rx_ring->next_to_use, rx_ring->next_to_clean);
        }

        /* Print RX Rings */
        if (!netif_msg_rx_status(adapter))
                return;

        dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

        /* Receive Descriptor Formats
         *
         * 82598 Advanced Receive Descriptor (Read) Format
         *    63                                           1        0
         *    +-----------------------------------------------------+
         *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
         *    +----------------------------------------------+------+
         *  8 |       Header Buffer Address [63:1]           |  DD  |
         *    +-----------------------------------------------------+
         *
         *
         * 82598 Advanced Receive Descriptor (Write-Back) Format
         *
         *   63       48 47    32 31  30      21 20 16 15   4 3     0
         *   +------------------------------------------------------+
         * 0 |       RSS Hash /  |SPH| HDR_LEN  | RSV |Packet|  RSS |
         *   | Packet   | IP     |   |          |     | Type | Type |
         *   | Checksum | Ident  |   |          |     |      |      |
         *   +------------------------------------------------------+
         * 8 | VLAN Tag | Length | Extended Error | Extended Status |
         *   +------------------------------------------------------+
         *   63       48 47    32 31            20 19               0
         *
         * 82599+ Advanced Receive Descriptor (Read) Format
         *    63                                           1        0
         *    +-----------------------------------------------------+
         *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
         *    +----------------------------------------------+------+
         *  8 |       Header Buffer Address [63:1]           |  DD  |
         *    +-----------------------------------------------------+
         *
         *
         * 82599+ Advanced Receive Descriptor (Write-Back) Format
         *
         *   63       48 47    32 31  30      21 20 17 16   4 3     0
         *   +------------------------------------------------------+
         * 0 |RSS / Frag Checksum|SPH| HDR_LEN  |RSC- |Packet|  RSS |
         *   |/ RTT / PCoE_PARAM |   |          | CNT | Type | Type |
         *   |/ Flow Dir Flt ID  |   |          |     |      |      |
         *   +------------------------------------------------------+
         * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
         *   +------------------------------------------------------+
         *   63       48 47    32 31          20 19                 0
         */

        for (n = 0; n < adapter->num_rx_queues; n++) {
                rx_ring = adapter->rx_ring[n];
                pr_info("------------------------------------\n");
                pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
                pr_info("------------------------------------\n");
                pr_info("%s%s%s\n",
                        "R  [desc]      [ PktBuf     A0] ",
                        "[  HeadBuf   DD] [bi->dma       ] [bi->skb       ] ",
                        "<-- Adv Rx Read format");
                pr_info("%s%s%s\n",
                        "RWB[desc]      [PcsmIpSHl PtRs] ",
                        "[vl er S cks ln] ---------------- [bi->skb       ] ",
                        "<-- Adv Rx Write-Back format");

                for (i = 0; i < rx_ring->count; i++) {
                        const char *ring_desc;

                        if (i == rx_ring->next_to_use)
                                ring_desc = " NTU";
                        else if (i == rx_ring->next_to_clean)
                                ring_desc = " NTC";
                        else
                                ring_desc = "";

                        rx_buffer_info = &rx_ring->rx_buffer_info[i];
                        rx_desc = IXGBE_RX_DESC(rx_ring, i);
                        u0 = (struct my_u0 *)rx_desc;
                        if (rx_desc->wb.upper.length) {
                                /* Descriptor Done */
                                pr_info("RWB[0x%03X]     %016llX %016llX ---------------- %p%s\n",
                                        i,
                                        le64_to_cpu((__force __le64)u0->a),
                                        le64_to_cpu((__force __le64)u0->b),
                                        rx_buffer_info->skb,
                                        ring_desc);
                        } else {
                                pr_info("R  [0x%03X]     %016llX %016llX %016llX %p%s\n",
                                        i,
                                        le64_to_cpu((__force __le64)u0->a),
                                        le64_to_cpu((__force __le64)u0->b),
                                        (u64)rx_buffer_info->dma,
                                        rx_buffer_info->skb,
                                        ring_desc);

                                if (netif_msg_pktdata(adapter) &&
                                    rx_buffer_info->dma) {
                                        print_hex_dump(KERN_INFO, "",
                                           DUMP_PREFIX_ADDRESS, 16, 1,
                                           page_address(rx_buffer_info->page) +
                                                    rx_buffer_info->page_offset,
                                           ixgbe_rx_bufsz(rx_ring), true);
                                }
                        }
                }
        }
}

static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
{
        u32 ctrl_ext;

        /* Let firmware take over control of h/w */
        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
                        ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
}

static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
{
        u32 ctrl_ext;

        /* Let firmware know the driver has taken over */
        ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
                        ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
}

/**
 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 * @adapter: pointer to adapter struct
 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 * @queue: queue to map the corresponding interrupt to
 * @msix_vector: the vector to map to the corresponding queue
 *
 */
static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
                           u8 queue, u8 msix_vector)
{
        u32 ivar, index;
        struct ixgbe_hw *hw = &adapter->hw;
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                if (direction == -1)
                        direction = 0;
                index = (((direction * 64) + queue) >> 2) & 0x1F;
                ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
                ivar &= ~(0xFF << (8 * (queue & 0x3)));
                ivar |= (msix_vector << (8 * (queue & 0x3)));
                IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (direction == -1) {
                        /* other causes */
                        msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                        index = ((queue & 1) * 8);
                        ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
                        ivar &= ~(0xFF << index);
                        ivar |= (msix_vector << index);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
                        break;
                } else {
                        /* tx or rx causes */
                        msix_vector |= IXGBE_IVAR_ALLOC_VAL;
                        index = ((16 * (queue & 1)) + (8 * direction));
                        ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
                        ivar &= ~(0xFF << index);
                        ivar |= (msix_vector << index);
                        IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
                        break;
                }
        default:
                break;
        }
}

void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
                            u64 qmask)
{
        u32 mask;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                mask = (qmask & 0xFFFFFFFF);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
                mask = (qmask >> 32);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
                break;
        default:
                break;
        }
}

static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_hw_stats *hwstats = &adapter->stats;
        int i;
        u32 data;

        if ((hw->fc.current_mode != ixgbe_fc_full) &&
            (hw->fc.current_mode != ixgbe_fc_rx_pause))
                return;

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
                break;
        default:
                data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
        }
        hwstats->lxoffrxc += data;

        /* refill credits (no tx hang) if we received xoff */
        if (!data)
                return;

        for (i = 0; i < adapter->num_tx_queues; i++)
                clear_bit(__IXGBE_HANG_CHECK_ARMED,
                          &adapter->tx_ring[i]->state);

        for (i = 0; i < adapter->num_xdp_queues; i++)
                clear_bit(__IXGBE_HANG_CHECK_ARMED,
                          &adapter->xdp_ring[i]->state);
}

static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_hw_stats *hwstats = &adapter->stats;
        u32 xoff[8] = {0};
        u8 tc;
        int i;
        bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

        if (adapter->ixgbe_ieee_pfc)
                pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

        if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
                ixgbe_update_xoff_rx_lfc(adapter);
                return;
        }

        /* update stats for each tc, only valid with PFC enabled */
        for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
                u32 pxoffrxc;

                switch (hw->mac.type) {
                case ixgbe_mac_82598EB:
                        pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
                        break;
                default:
                        pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
                }
                hwstats->pxoffrxc[i] += pxoffrxc;
                /* Get the TC for given UP */
                tc = netdev_get_prio_tc_map(adapter->netdev, i);
                xoff[tc] += pxoffrxc;
        }

        /* disarm tx queues that have received xoff frames */
        for (i = 0; i < adapter->num_tx_queues; i++) {
                struct ixgbe_ring *tx_ring = adapter->tx_ring[i];

                tc = tx_ring->dcb_tc;
                if (xoff[tc])
                        clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
        }

        for (i = 0; i < adapter->num_xdp_queues; i++) {

                struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];

                tc = xdp_ring->dcb_tc;
                if (xoff[tc])
                        clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);
        }
}

static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
{
        return ring->stats.packets;
}

static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
{
        unsigned int head, tail;

        head = ring->next_to_clean;
        tail = ring->next_to_use;

        return ((head <= tail) ? tail : tail + ring->count) - head;
}

static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
{
        u32 tx_done = ixgbe_get_tx_completed(tx_ring);
        u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
        u32 tx_pending = ixgbe_get_tx_pending(tx_ring);

        clear_check_for_tx_hang(tx_ring);

        /*
         * Check for a hung queue, but be thorough. This verifies
         * that a transmit has been completed since the previous
         * check AND there is at least one packet pending. The
         * ARMED bit is set to indicate a potential hang. The
         * bit is cleared if a pause frame is received to remove
         * false hang detection due to PFC or 802.3x frames. By
         * requiring this to fail twice we avoid races with
         * pfc clearing the ARMED bit and conditions where we
         * run the check_tx_hang logic with a transmit completion
         * pending but without time to complete it yet.
         */
        if (tx_done_old == tx_done && tx_pending)
                /* make sure it is true for two checks in a row */
                return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
                                        &tx_ring->state);
        /* update completed stats and continue */
        tx_ring->tx_stats.tx_done_old = tx_done;
        /* reset the countdown */
        clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);

        return false;
}

/**
 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 * @adapter: driver private struct
 **/
static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
{

        /* Do the reset outside of interrupt context */
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                e_warn(drv, "initiating reset due to tx timeout\n");
                ixgbe_service_event_schedule(adapter);
        }
}

/**
 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
 * @netdev: network interface device structure
 * @queue_index: Tx queue to set
 * @maxrate: desired maximum transmit bitrate
 **/
static int ixgbe_tx_maxrate(struct net_device *netdev,
                            int queue_index, u32 maxrate)
{
        struct ixgbe_adapter *adapter = netdev_priv(netdev);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 bcnrc_val = ixgbe_link_mbps(adapter);

        if (!maxrate)
                return 0;

        /* Calculate the rate factor values to set */
        bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
        bcnrc_val /= maxrate;

        /* clear everything but the rate factor */
        bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
        IXGBE_RTTBCNRC_RF_DEC_MASK;

        /* enable the rate scheduler */
        bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;

        IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
        IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);

        return 0;
}

/**
 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
 * @q_vector: structure containing interrupt and ring information
 * @tx_ring: tx ring to clean
 * @napi_budget: Used to determine if we are in netpoll
 **/
static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
                               struct ixgbe_ring *tx_ring, int napi_budget)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_tx_buffer *tx_buffer;
        union ixgbe_adv_tx_desc *tx_desc;
        unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
        unsigned int budget = q_vector->tx.work_limit;
        unsigned int i = tx_ring->next_to_clean;

        if (test_bit(__IXGBE_DOWN, &adapter->state))
                return true;

        tx_buffer = &tx_ring->tx_buffer_info[i];
        tx_desc = IXGBE_TX_DESC(tx_ring, i);
        i -= tx_ring->count;

        do {
                union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;

                /* if next_to_watch is not set then there is no work pending */
                if (!eop_desc)
                        break;

                /* prevent any other reads prior to eop_desc */
                smp_rmb();

                /* if DD is not set pending work has not been completed */
                if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
                        break;

                /* clear next_to_watch to prevent false hangs */
                tx_buffer->next_to_watch = NULL;

                /* update the statistics for this packet */
                total_bytes += tx_buffer->bytecount;
                total_packets += tx_buffer->gso_segs;
                if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
                        total_ipsec++;

                /* free the skb */
                if (ring_is_xdp(tx_ring))
                        xdp_return_frame(tx_buffer->xdpf);
                else
                        napi_consume_skb(tx_buffer->skb, napi_budget);

                /* unmap skb header data */
                dma_unmap_single(tx_ring->dev,
                                 dma_unmap_addr(tx_buffer, dma),
                                 dma_unmap_len(tx_buffer, len),
                                 DMA_TO_DEVICE);

                /* clear tx_buffer data */
                dma_unmap_len_set(tx_buffer, len, 0);

                /* unmap remaining buffers */
                while (tx_desc != eop_desc) {
                        tx_buffer++;
                        tx_desc++;
                        i++;
                        if (unlikely(!i)) {
                                i -= tx_ring->count;
                                tx_buffer = tx_ring->tx_buffer_info;
                                tx_desc = IXGBE_TX_DESC(tx_ring, 0);
                        }

                        /* unmap any remaining paged data */
                        if (dma_unmap_len(tx_buffer, len)) {
                                dma_unmap_page(tx_ring->dev,
                                               dma_unmap_addr(tx_buffer, dma),
                                               dma_unmap_len(tx_buffer, len),
                                               DMA_TO_DEVICE);
                                dma_unmap_len_set(tx_buffer, len, 0);
                        }
                }

                /* move us one more past the eop_desc for start of next pkt */
                tx_buffer++;
                tx_desc++;
                i++;
                if (unlikely(!i)) {
                        i -= tx_ring->count;
                        tx_buffer = tx_ring->tx_buffer_info;
                        tx_desc = IXGBE_TX_DESC(tx_ring, 0);
                }

                /* issue prefetch for next Tx descriptor */
                prefetch(tx_desc);

                /* update budget accounting */
                budget--;
        } while (likely(budget));

        i += tx_ring->count;
        tx_ring->next_to_clean = i;
        u64_stats_update_begin(&tx_ring->syncp);
        tx_ring->stats.bytes += total_bytes;
        tx_ring->stats.packets += total_packets;
        u64_stats_update_end(&tx_ring->syncp);
        q_vector->tx.total_bytes += total_bytes;
        q_vector->tx.total_packets += total_packets;
        adapter->tx_ipsec += total_ipsec;

        if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
                /* schedule immediate reset if we believe we hung */
                struct ixgbe_hw *hw = &adapter->hw;
                e_err(drv, "Detected Tx Unit Hang %s\n"
                        "  Tx Queue             <%d>\n"
                        "  TDH, TDT             <%x>, <%x>\n"
                        "  next_to_use          <%x>\n"
                        "  next_to_clean        <%x>\n"
                        "tx_buffer_info[next_to_clean]\n"
                        "  time_stamp           <%lx>\n"
                        "  jiffies              <%lx>\n",
                        ring_is_xdp(tx_ring) ? "(XDP)" : "",
                        tx_ring->queue_index,
                        IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
                        IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
                        tx_ring->next_to_use, i,
                        tx_ring->tx_buffer_info[i].time_stamp, jiffies);

                if (!ring_is_xdp(tx_ring))
                        netif_stop_subqueue(tx_ring->netdev,
                                            tx_ring->queue_index);

                e_info(probe,
                       "tx hang %d detected on queue %d, resetting adapter\n",
                        adapter->tx_timeout_count + 1, tx_ring->queue_index);

                /* schedule immediate reset if we believe we hung */
                ixgbe_tx_timeout_reset(adapter);

                /* the adapter is about to reset, no point in enabling stuff */
                return true;
        }

        if (ring_is_xdp(tx_ring))
                return !!budget;

        netdev_tx_completed_queue(txring_txq(tx_ring),
                                  total_packets, total_bytes);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
        if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
                     (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
                /* Make sure that anybody stopping the queue after this
                 * sees the new next_to_clean.
                 */
                smp_mb();
                if (__netif_subqueue_stopped(tx_ring->netdev,
                                             tx_ring->queue_index)
                    && !test_bit(__IXGBE_DOWN, &adapter->state)) {
                        netif_wake_subqueue(tx_ring->netdev,
                                            tx_ring->queue_index);
                        ++tx_ring->tx_stats.restart_queue;
                }
        }

        return !!budget;
}

#ifdef CONFIG_IXGBE_DCA
static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
                                struct ixgbe_ring *tx_ring,
                                int cpu)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 txctrl = 0;
        u16 reg_offset;

        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                txctrl = dca3_get_tag(tx_ring->dev, cpu);

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
                reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
                txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
                break;
        default:
                /* for unknown hardware do not write register */
                return;
        }

        /*
         * We can enable relaxed ordering for reads, but not writes when
         * DCA is enabled.  This is due to a known issue in some chipsets
         * which will cause the DCA tag to be cleared.
         */
        txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
                  IXGBE_DCA_TXCTRL_DATA_RRO_EN |
                  IXGBE_DCA_TXCTRL_DESC_DCA_EN;

        IXGBE_WRITE_REG(hw, reg_offset, txctrl);
}

static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
                                struct ixgbe_ring *rx_ring,
                                int cpu)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 rxctrl = 0;
        u8 reg_idx = rx_ring->reg_idx;

        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                rxctrl = dca3_get_tag(rx_ring->dev, cpu);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
                rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
                break;
        default:
                break;
        }

        /*
         * We can enable relaxed ordering for reads, but not writes when
         * DCA is enabled.  This is due to a known issue in some chipsets
         * which will cause the DCA tag to be cleared.
         */
        rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
                  IXGBE_DCA_RXCTRL_DATA_DCA_EN |
                  IXGBE_DCA_RXCTRL_DESC_DCA_EN;

        IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
}

static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_ring *ring;
        int cpu = get_cpu();

        if (q_vector->cpu == cpu)
                goto out_no_update;

        ixgbe_for_each_ring(ring, q_vector->tx)
                ixgbe_update_tx_dca(adapter, ring, cpu);

        ixgbe_for_each_ring(ring, q_vector->rx)
                ixgbe_update_rx_dca(adapter, ring, cpu);

        q_vector->cpu = cpu;
out_no_update:
        put_cpu();
}

static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
{
        int i;

        /* always use CB2 mode, difference is masked in the CB driver */
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                IXGBE_DCA_CTRL_DCA_MODE_CB2);
        else
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                IXGBE_DCA_CTRL_DCA_DISABLE);

        for (i = 0; i < adapter->num_q_vectors; i++) {
                adapter->q_vector[i]->cpu = -1;
                ixgbe_update_dca(adapter->q_vector[i]);
        }
}

static int __ixgbe_notify_dca(struct device *dev, void *data)
{
        struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
        unsigned long event = *(unsigned long *)data;

        if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
                return 0;

        switch (event) {
        case DCA_PROVIDER_ADD:
                /* if we're already enabled, don't do it again */
                if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                        break;
                if (dca_add_requester(dev) == 0) {
                        adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                        IXGBE_DCA_CTRL_DCA_MODE_CB2);
                        break;
                }
                fallthrough; /* DCA is disabled. */
        case DCA_PROVIDER_REMOVE:
                if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
                        dca_remove_requester(dev);
                        adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
                                        IXGBE_DCA_CTRL_DCA_DISABLE);
                }
                break;
        }

        return 0;
}

#endif /* CONFIG_IXGBE_DCA */

#define IXGBE_RSS_L4_TYPES_MASK \
        ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
         (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))

static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
                                 union ixgbe_adv_rx_desc *rx_desc,
                                 struct sk_buff *skb)
{
        u16 rss_type;

        if (!(ring->netdev->features & NETIF_F_RXHASH))
                return;

        rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
                   IXGBE_RXDADV_RSSTYPE_MASK;

        if (!rss_type)
                return;

        skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
                     (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
                     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
}

#ifdef IXGBE_FCOE
/**
 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
 * @ring: structure containing ring specific data
 * @rx_desc: advanced rx descriptor
 *
 * Returns : true if it is FCoE pkt
 */
static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
                                    union ixgbe_adv_rx_desc *rx_desc)
{
        __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

        return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
               ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
                (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
                             IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
}

#endif /* IXGBE_FCOE */
/**
 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
 * @ring: structure containing ring specific data
 * @rx_desc: current Rx descriptor being processed
 * @skb: skb currently being received and modified
 **/
static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
                                     union ixgbe_adv_rx_desc *rx_desc,
                                     struct sk_buff *skb)
{
        __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
        bool encap_pkt = false;

        skb_checksum_none_assert(skb);

        /* Rx csum disabled */
        if (!(ring->netdev->features & NETIF_F_RXCSUM))
                return;

        /* check for VXLAN and Geneve packets */
        if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
                encap_pkt = true;
                skb->encapsulation = 1;
        }

        /* if IP and error */
        if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
            ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
                ring->rx_stats.csum_err++;
                return;
        }

        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
                return;

        if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
                /*
                 * 82599 errata, UDP frames with a 0 checksum can be marked as
                 * checksum errors.
                 */
                if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
                    test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
                        return;

                ring->rx_stats.csum_err++;
                return;
        }

        /* It must be a TCP or UDP packet with a valid checksum */
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        if (encap_pkt) {
                if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
                        return;

                if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
                        skb->ip_summed = CHECKSUM_NONE;
                        return;
                }
                /* If we checked the outer header let the stack know */
                skb->csum_level = 1;
        }
}

static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring)
{
        return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0;
}

static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
                                    struct ixgbe_rx_buffer *bi)
{
        struct page *page = bi->page;
        dma_addr_t dma;

        /* since we are recycling buffers we should seldom need to alloc */
        if (likely(page))
                return true;

        /* alloc new page for storage */
        page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
        if (unlikely(!page)) {
                rx_ring->rx_stats.alloc_rx_page_failed++;
                return false;
        }

        /* map page for use */
        dma = dma_map_page_attrs(rx_ring->dev, page, 0,
                                 ixgbe_rx_pg_size(rx_ring),
                                 DMA_FROM_DEVICE,
                                 IXGBE_RX_DMA_ATTR);

        /*
         * if mapping failed free memory back to system since
         * there isn't much point in holding memory we can't use
         */
        if (dma_mapping_error(rx_ring->dev, dma)) {
                __free_pages(page, ixgbe_rx_pg_order(rx_ring));

                rx_ring->rx_stats.alloc_rx_page_failed++;
                return false;
        }

        bi->dma = dma;
        bi->page = page;
        bi->page_offset = ixgbe_rx_offset(rx_ring);
        page_ref_add(page, USHRT_MAX - 1);
        bi->pagecnt_bias = USHRT_MAX;
        rx_ring->rx_stats.alloc_rx_page++;

        return true;
}

/**
 * ixgbe_alloc_rx_buffers - Replace used receive buffers
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
 **/
void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
{
        union ixgbe_adv_rx_desc *rx_desc;
        struct ixgbe_rx_buffer *bi;
        u16 i = rx_ring->next_to_use;
        u16 bufsz;

        /* nothing to do */
        if (!cleaned_count)
                return;

        rx_desc = IXGBE_RX_DESC(rx_ring, i);
        bi = &rx_ring->rx_buffer_info[i];
        i -= rx_ring->count;

        bufsz = ixgbe_rx_bufsz(rx_ring);

        do {
                if (!ixgbe_alloc_mapped_page(rx_ring, bi))
                        break;

                /* sync the buffer for use by the device */
                dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
                                                 bi->page_offset, bufsz,
                                                 DMA_FROM_DEVICE);

                /*
                 * Refresh the desc even if buffer_addrs didn't change
                 * because each write-back erases this info.
                 */
                rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);

                rx_desc++;
                bi++;
                i++;
                if (unlikely(!i)) {
                        rx_desc = IXGBE_RX_DESC(rx_ring, 0);
                        bi = rx_ring->rx_buffer_info;
                        i -= rx_ring->count;
                }

                /* clear the length for the next_to_use descriptor */
                rx_desc->wb.upper.length = 0;

                cleaned_count--;
        } while (cleaned_count);

        i += rx_ring->count;

        if (rx_ring->next_to_use != i) {
                rx_ring->next_to_use = i;

                /* update next to alloc since we have filled the ring */
                rx_ring->next_to_alloc = i;

                /* Force memory writes to complete before letting h/w
                 * know there are new descriptors to fetch.  (Only
                 * applicable for weak-ordered memory model archs,
                 * such as IA-64).
                 */
                wmb();
                writel(i, rx_ring->tail);
        }
}

static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
                                   struct sk_buff *skb)
{
        u16 hdr_len = skb_headlen(skb);

        /* set gso_size to avoid messing up TCP MSS */
        skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
                                                 IXGBE_CB(skb)->append_cnt);
        skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
}

static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
                                   struct sk_buff *skb)
{
        /* if append_cnt is 0 then frame is not RSC */
        if (!IXGBE_CB(skb)->append_cnt)
                return;

        rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
        rx_ring->rx_stats.rsc_flush++;

        ixgbe_set_rsc_gso_size(rx_ring, skb);

        /* gso_size is computed using append_cnt so always clear it last */
        IXGBE_CB(skb)->append_cnt = 0;
}

/**
 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being populated
 *
 * This function checks the ring, descriptor, and packet information in
 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
 * other fields within the skb.
 **/
void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
                              union ixgbe_adv_rx_desc *rx_desc,
                              struct sk_buff *skb)
{
        struct net_device *dev = rx_ring->netdev;
        u32 flags = rx_ring->q_vector->adapter->flags;

        ixgbe_update_rsc_stats(rx_ring, skb);

        ixgbe_rx_hash(rx_ring, rx_desc, skb);

        ixgbe_rx_checksum(rx_ring, rx_desc, skb);

        if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
                ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);

        if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
            ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
                u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
        }

        if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
                ixgbe_ipsec_rx(rx_ring, rx_desc, skb);

        /* record Rx queue, or update MACVLAN statistics */
        if (netif_is_ixgbe(dev))
                skb_record_rx_queue(skb, rx_ring->queue_index);
        else
                macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
                                 false);

        skb->protocol = eth_type_trans(skb, dev);
}

void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
                  struct sk_buff *skb)
{
        napi_gro_receive(&q_vector->napi, skb);
}

/**
 * ixgbe_is_non_eop - process handling of non-EOP buffers
 * @rx_ring: Rx ring being processed
 * @rx_desc: Rx descriptor for current buffer
 * @skb: Current socket buffer containing buffer in progress
 *
 * This function updates next to clean.  If the buffer is an EOP buffer
 * this function exits returning false, otherwise it will place the
 * sk_buff in the next buffer to be chained and return true indicating
 * that this is in fact a non-EOP buffer.
 **/
static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
                             union ixgbe_adv_rx_desc *rx_desc,
                             struct sk_buff *skb)
{
        u32 ntc = rx_ring->next_to_clean + 1;

        /* fetch, update, and store next to clean */
        ntc = (ntc < rx_ring->count) ? ntc : 0;
        rx_ring->next_to_clean = ntc;

        prefetch(IXGBE_RX_DESC(rx_ring, ntc));

        /* update RSC append count if present */
        if (ring_is_rsc_enabled(rx_ring)) {
                __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
                                     cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);

                if (unlikely(rsc_enabled)) {
                        u32 rsc_cnt = le32_to_cpu(rsc_enabled);

                        rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
                        IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;

                        /* update ntc based on RSC value */
                        ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
                        ntc &= IXGBE_RXDADV_NEXTP_MASK;
                        ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
                }
        }

        /* if we are the last buffer then there is nothing else to do */
        if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
                return false;

        /* place skb in next buffer to be received */
        rx_ring->rx_buffer_info[ntc].skb = skb;
        rx_ring->rx_stats.non_eop_descs++;

        return true;
}

/**
 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being adjusted
 *
 * This function is an ixgbe specific version of __pskb_pull_tail.  The
 * main difference between this version and the original function is that
 * this function can make several assumptions about the state of things
 * that allow for significant optimizations versus the standard function.
 * As a result we can do things like drop a frag and maintain an accurate
 * truesize for the skb.
 */
static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
                            struct sk_buff *skb)
{
        skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
        unsigned char *va;
        unsigned int pull_len;

        /*
         * it is valid to use page_address instead of kmap since we are
         * working with pages allocated out of the lomem pool per
         * alloc_page(GFP_ATOMIC)
         */
        va = skb_frag_address(frag);

        /*
         * we need the header to contain the greater of either ETH_HLEN or
         * 60 bytes if the skb->len is less than 60 for skb_pad.
         */
        pull_len = eth_get_headlen(skb->dev, va, IXGBE_RX_HDR_SIZE);

        /* align pull length to size of long to optimize memcpy performance */
        skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

        /* update all of the pointers */
        skb_frag_size_sub(frag, pull_len);
        skb_frag_off_add(frag, pull_len);
        skb->data_len -= pull_len;
        skb->tail += pull_len;
}

/**
 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being updated
 *
 * This function provides a basic DMA sync up for the first fragment of an
 * skb.  The reason for doing this is that the first fragment cannot be
 * unmapped until we have reached the end of packet descriptor for a buffer
 * chain.
 */
static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
                                struct sk_buff *skb)
{
        if (ring_uses_build_skb(rx_ring)) {
                unsigned long offset = (unsigned long)(skb->data) & ~PAGE_MASK;

                dma_sync_single_range_for_cpu(rx_ring->dev,
                                              IXGBE_CB(skb)->dma,
                                              offset,
                                              skb_headlen(skb),
                                              DMA_FROM_DEVICE);
        } else {
                skb_frag_t *frag = &skb_shinfo(skb)->frags[0];

                dma_sync_single_range_for_cpu(rx_ring->dev,
                                              IXGBE_CB(skb)->dma,
                                              skb_frag_off(frag),
                                              skb_frag_size(frag),
                                              DMA_FROM_DEVICE);
        }

        /* If the page was released, just unmap it. */
        if (unlikely(IXGBE_CB(skb)->page_released)) {
                dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma,
                                     ixgbe_rx_pg_size(rx_ring),
                                     DMA_FROM_DEVICE,
                                     IXGBE_RX_DMA_ATTR);
        }
}

/**
 * ixgbe_cleanup_headers - Correct corrupted or empty headers
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being fixed
 *
 * Check if the skb is valid in the XDP case it will be an error pointer.
 * Return true in this case to abort processing and advance to next
 * descriptor.
 *
 * Check for corrupted packet headers caused by senders on the local L2
 * embedded NIC switch not setting up their Tx Descriptors right.  These
 * should be very rare.
 *
 * Also address the case where we are pulling data in on pages only
 * and as such no data is present in the skb header.
 *
 * In addition if skb is not at least 60 bytes we need to pad it so that
 * it is large enough to qualify as a valid Ethernet frame.
 *
 * Returns true if an error was encountered and skb was freed.
 **/
bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
                           union ixgbe_adv_rx_desc *rx_desc,
                           struct sk_buff *skb)
{
        struct net_device *netdev = rx_ring->netdev;

        /* XDP packets use error pointer so abort at this point */
        if (IS_ERR(skb))
                return true;

        /* Verify netdev is present, and that packet does not have any
         * errors that would be unacceptable to the netdev.
         */
        if (!netdev ||
            (unlikely(ixgbe_test_staterr(rx_desc,
                                         IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
             !(netdev->features & NETIF_F_RXALL)))) {
                dev_kfree_skb_any(skb);
                return true;
        }

        /* place header in linear portion of buffer */
        if (!skb_headlen(skb))
                ixgbe_pull_tail(rx_ring, skb);

#ifdef IXGBE_FCOE
        /* do not attempt to pad FCoE Frames as this will disrupt DDP */
        if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
                return false;

#endif
        /* if eth_skb_pad returns an error the skb was freed */
        if (eth_skb_pad(skb))
                return true;

        return false;
}

/**
 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
 * @rx_ring: rx descriptor ring to store buffers on
 * @old_buff: donor buffer to have page reused
 *
 * Synchronizes page for reuse by the adapter
 **/
static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
                                struct ixgbe_rx_buffer *old_buff)
{
        struct ixgbe_rx_buffer *new_buff;
        u16 nta = rx_ring->next_to_alloc;

        new_buff = &rx_ring->rx_buffer_info[nta];

        /* update, and store next to alloc */
        nta++;
        rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

        /* Transfer page from old buffer to new buffer.
         * Move each member individually to avoid possible store
         * forwarding stalls and unnecessary copy of skb.
         */
        new_buff->dma           = old_buff->dma;
        new_buff->page          = old_buff->page;
        new_buff->page_offset   = old_buff->page_offset;
        new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
}

static inline bool ixgbe_page_is_reserved(struct page *page)
{
        return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
}

static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer,
                                    int rx_buffer_pgcnt)
{
        unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
        struct page *page = rx_buffer->page;

        /* avoid re-using remote pages */
        if (unlikely(ixgbe_page_is_reserved(page)))
                return false;

#if (PAGE_SIZE < 8192)
        /* if we are only owner of page we can reuse it */
        if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
                return false;
#else
        /* The last offset is a bit aggressive in that we assume the
         * worst case of FCoE being enabled and using a 3K buffer.
         * However this should have minimal impact as the 1K extra is
         * still less than one buffer in size.
         */
#define IXGBE_LAST_OFFSET \
        (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
        if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
                return false;
#endif

        /* If we have drained the page fragment pool we need to update
         * the pagecnt_bias and page count so that we fully restock the
         * number of references the driver holds.
         */
        if (unlikely(pagecnt_bias == 1)) {
                page_ref_add(page, USHRT_MAX - 1);
                rx_buffer->pagecnt_bias = USHRT_MAX;
        }

        return true;
}

/**
 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_buffer: buffer containing page to add
 * @skb: sk_buff to place the data into
 * @size: size of data in rx_buffer
 *
 * This function will add the data contained in rx_buffer->page to the skb.
 * This is done either through a direct copy if the data in the buffer is
 * less than the skb header size, otherwise it will just attach the page as
 * a frag to the skb.
 *
 * The function will then update the page offset if necessary and return
 * true if the buffer can be reused by the adapter.
 **/

static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
                              struct ixgbe_rx_buffer *rx_buffer,
                              struct sk_buff *skb,
                              unsigned int size)
{
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = ring_uses_build_skb(rx_ring) ?
                                SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) :
                                SKB_DATA_ALIGN(size);
#endif
        skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
                        rx_buffer->page_offset, size, truesize);
#if (PAGE_SIZE < 8192)
        rx_buffer->page_offset ^= truesize;
#else
        rx_buffer->page_offset += truesize;
#endif
}

static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
                                                   union ixgbe_adv_rx_desc *rx_desc,
                                                   struct sk_buff **skb,
                                                   const unsigned int size,
                                                   int *rx_buffer_pgcnt)
{
        struct ixgbe_rx_buffer *rx_buffer;

        rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
        *rx_buffer_pgcnt =
#if (PAGE_SIZE < 8192)
                page_count(rx_buffer->page);
#else
                0;
#endif
        prefetchw(rx_buffer->page);
        *skb = rx_buffer->skb;

        /* Delay unmapping of the first packet. It carries the header
         * information, HW may still access the header after the writeback.
         * Only unmap it when EOP is reached
         */
        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
                if (!*skb)
                        goto skip_sync;
        } else {
                if (*skb)
                        ixgbe_dma_sync_frag(rx_ring, *skb);
        }

        /* we are reusing so sync this buffer for CPU use */
        dma_sync_single_range_for_cpu(rx_ring->dev,
                                      rx_buffer->dma,
                                      rx_buffer->page_offset,
                                      size,
                                      DMA_FROM_DEVICE);
skip_sync:
        rx_buffer->pagecnt_bias--;

        return rx_buffer;
}

static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
                                struct ixgbe_rx_buffer *rx_buffer,
                                struct sk_buff *skb,
                                int rx_buffer_pgcnt)
{
        if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
                /* hand second half of page back to the ring */
                ixgbe_reuse_rx_page(rx_ring, rx_buffer);
        } else {
                if (!IS_ERR(skb) && IXGBE_CB(skb)->dma == rx_buffer->dma) {
                        /* the page has been released from the ring */
                        IXGBE_CB(skb)->page_released = true;
                } else {
                        /* we are not reusing the buffer so unmap it */
                        dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
                                             ixgbe_rx_pg_size(rx_ring),
                                             DMA_FROM_DEVICE,
                                             IXGBE_RX_DMA_ATTR);
                }
                __page_frag_cache_drain(rx_buffer->page,
                                        rx_buffer->pagecnt_bias);
        }

        /* clear contents of rx_buffer */
        rx_buffer->page = NULL;
        rx_buffer->skb = NULL;
}

static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
                                           struct ixgbe_rx_buffer *rx_buffer,
                                           struct xdp_buff *xdp,
                                           union ixgbe_adv_rx_desc *rx_desc)
{
        unsigned int size = xdp->data_end - xdp->data;
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
                                               xdp->data_hard_start);
#endif
        struct sk_buff *skb;

        /* prefetch first cache line of first page */
        net_prefetch(xdp->data);

        /* Note, we get here by enabling legacy-rx via:
         *
         *    ethtool --set-priv-flags <dev> legacy-rx on
         *
         * In this mode, we currently get 0 extra XDP headroom as
         * opposed to having legacy-rx off, where we process XDP
         * packets going to stack via ixgbe_build_skb(). The latter
         * provides us currently with 192 bytes of headroom.
         *
         * For ixgbe_construct_skb() mode it means that the
         * xdp->data_meta will always point to xdp->data, since
         * the helper cannot expand the head. Should this ever
         * change in future for legacy-rx mode on, then lets also
         * add xdp->data_meta handling here.
         */

        /* allocate a skb to store the frags */
        skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
        if (unlikely(!skb))
                return NULL;

        if (size > IXGBE_RX_HDR_SIZE) {
                if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
                        IXGBE_CB(skb)->dma = rx_buffer->dma;

                skb_add_rx_frag(skb, 0, rx_buffer->page,
                                xdp->data - page_address(rx_buffer->page),
                                size, truesize);
#if (PAGE_SIZE < 8192)
                rx_buffer->page_offset ^= truesize;
#else
                rx_buffer->page_offset += truesize;
#endif
        } else {
                memcpy(__skb_put(skb, size),
                       xdp->data, ALIGN(size, sizeof(long)));
                rx_buffer->pagecnt_bias++;
        }

        return skb;
}

static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
                                       struct ixgbe_rx_buffer *rx_buffer,
                                       struct xdp_buff *xdp,
                                       union ixgbe_adv_rx_desc *rx_desc)
{
        unsigned int metasize = xdp->data - xdp->data_meta;
#if (PAGE_SIZE < 8192)
        unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
        unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
                                SKB_DATA_ALIGN(xdp->data_end -
                                               xdp->data_hard_start);
#endif
        struct sk_buff *skb;

        /* Prefetch first cache line of first page. If xdp->data_meta
         * is unused, this points extactly as xdp->data, otherwise we
         * likely have a consumer accessing first few bytes of meta
         * data, and then actual data.
         */
        net_prefetch(xdp->data_meta);

        /* build an skb to around the page buffer */
        skb = build_skb(xdp->data_hard_start, truesize);
        if (unlikely(!skb))
                return NULL;

        /* update pointers within the skb to store the data */
        skb_reserve(skb, xdp->data - xdp->data_hard_start);
        __skb_put(skb, xdp->data_end - xdp->data);
        if (metasize)
                skb_metadata_set(skb, metasize);

        /* record DMA address if this is the start of a chain of buffers */
        if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
                IXGBE_CB(skb)->dma = rx_buffer->dma;

        /* update buffer offset */
#if (PAGE_SIZE < 8192)
        rx_buffer->page_offset ^= truesize;
#else
        rx_buffer->page_offset += truesize;
#endif

        return skb;
}

static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
                                     struct ixgbe_ring *rx_ring,
                                     struct xdp_buff *xdp)
{
        int err, result = IXGBE_XDP_PASS;
        struct bpf_prog *xdp_prog;
        struct xdp_frame *xdpf;
        u32 act;

        rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);

        if (!xdp_prog)
                goto xdp_out;

        prefetchw(xdp->data_hard_start); /* xdp_frame write */

        act = bpf_prog_run_xdp(xdp_prog, xdp);
        switch (act) {
        case XDP_PASS:
                break;
        case XDP_TX:
                xdpf = xdp_convert_buff_to_frame(xdp);
                if (unlikely(!xdpf)) {
                        result = IXGBE_XDP_CONSUMED;
                        break;
                }
                result = ixgbe_xmit_xdp_ring(adapter, xdpf);
                break;
        case XDP_REDIRECT:
                err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
                if (!err)
                        result = IXGBE_XDP_REDIR;
                else
                        result = IXGBE_XDP_CONSUMED;
                break;
        default:
                bpf_warn_invalid_xdp_action(act);
                fallthrough;
        case XDP_ABORTED:
                trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
                fallthrough; /* handle aborts by dropping packet */
        case XDP_DROP:
                result = IXGBE_XDP_CONSUMED;
                break;
        }
xdp_out:
        rcu_read_unlock();
        return ERR_PTR(-result);
}

static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring,
                                            unsigned int size)
{
        unsigned int truesize;

#if (PAGE_SIZE < 8192)
        truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
#else
        truesize = ring_uses_build_skb(rx_ring) ?
                SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) +
                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
                SKB_DATA_ALIGN(size);
#endif
        return truesize;
}

static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring,
                                 struct ixgbe_rx_buffer *rx_buffer,
                                 unsigned int size)
{
        unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size);
#if (PAGE_SIZE < 8192)
        rx_buffer->page_offset ^= truesize;
#else
        rx_buffer->page_offset += truesize;
#endif
}

/**
 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 * @q_vector: structure containing interrupt and ring information
 * @rx_ring: rx descriptor ring to transact packets on
 * @budget: Total limit on number of packets to process
 *
 * This function provides a "bounce buffer" approach to Rx interrupt
 * processing.  The advantage to this is that on systems that have
 * expensive overhead for IOMMU access this provides a means of avoiding
 * it by maintaining the mapping of the page to the syste.
 *
 * Returns amount of work completed
 **/
static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
                               struct ixgbe_ring *rx_ring,
                               const int budget)
{
        unsigned int total_rx_bytes = 0, total_rx_packets = 0;
        struct ixgbe_adapter *adapter = q_vector->adapter;
#ifdef IXGBE_FCOE
        int ddp_bytes;
        unsigned int mss = 0;
#endif /* IXGBE_FCOE */
        u16 cleaned_count = ixgbe_desc_unused(rx_ring);
        unsigned int xdp_xmit = 0;
        struct xdp_buff xdp;

        xdp.rxq = &rx_ring->xdp_rxq;

        /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
#if (PAGE_SIZE < 8192)
        xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0);
#endif

        while (likely(total_rx_packets < budget)) {
                union ixgbe_adv_rx_desc *rx_desc;
                struct ixgbe_rx_buffer *rx_buffer;
                struct sk_buff *skb;
                int rx_buffer_pgcnt;
                unsigned int size;

                /* return some buffers to hardware, one at a time is too slow */
                if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
                        ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
                        cleaned_count = 0;
                }

                rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
                size = le16_to_cpu(rx_desc->wb.upper.length);
                if (!size)
                        break;

                /* This memory barrier is needed to keep us from reading
                 * any other fields out of the rx_desc until we know the
                 * descriptor has been written back
                 */
                dma_rmb();

                rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt);

                /* retrieve a buffer from the ring */
                if (!skb) {
                        xdp.data = page_address(rx_buffer->page) +
                                   rx_buffer->page_offset;
                        xdp.data_meta = xdp.data;
                        xdp.data_hard_start = xdp.data -
                                              ixgbe_rx_offset(rx_ring);
                        xdp.data_end = xdp.data + size;
#if (PAGE_SIZE > 4096)
                        /* At larger PAGE_SIZE, frame_sz depend on len size */
                        xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size);
#endif
                        skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);
                }

                if (IS_ERR(skb)) {
                        unsigned int xdp_res = -PTR_ERR(skb);

                        if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {
                                xdp_xmit |= xdp_res;
                                ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);
                        } else {
                                rx_buffer->pagecnt_bias++;
                        }
                        total_rx_packets++;
                        total_rx_bytes += size;
                } else if (skb) {
                        ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
                } else if (ring_uses_build_skb(rx_ring)) {
                        skb = ixgbe_build_skb(rx_ring, rx_buffer,
                                              &xdp, rx_desc);
                } else {
                        skb = ixgbe_construct_skb(rx_ring, rx_buffer,
                                                  &xdp, rx_desc);
                }

                /* exit if we failed to retrieve a buffer */
                if (!skb) {
                        rx_ring->rx_stats.alloc_rx_buff_failed++;
                        rx_buffer->pagecnt_bias++;
                        break;
                }

                ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
                cleaned_count++;

                /* place incomplete frames back on ring for completion */
                if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
                        continue;

                /* verify the packet layout is correct */
                if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
                        continue;

                /* probably a little skewed due to removing CRC */
                total_rx_bytes += skb->len;

                /* populate checksum, timestamp, VLAN, and protocol */
                ixgbe_process_skb_fields(rx_ring, rx_desc, skb);

#ifdef IXGBE_FCOE
                /* if ddp, not passing to ULD unless for FCP_RSP or error */
                if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
                        ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
                        /* include DDPed FCoE data */
                        if (ddp_bytes > 0) {
                                if (!mss) {
                                        mss = rx_ring->netdev->mtu -
                                                sizeof(struct fcoe_hdr) -
                                                sizeof(struct fc_frame_header) -
                                                sizeof(struct fcoe_crc_eof);
                                        if (mss > 512)
                                                mss &= ~511;
                                }
                                total_rx_bytes += ddp_bytes;
                                total_rx_packets += DIV_ROUND_UP(ddp_bytes,
                                                                 mss);
                        }
                        if (!ddp_bytes) {
                                dev_kfree_skb_any(skb);
                                continue;
                        }
                }

#endif /* IXGBE_FCOE */
                ixgbe_rx_skb(q_vector, skb);

                /* update budget accounting */
                total_rx_packets++;
        }

        if (xdp_xmit & IXGBE_XDP_REDIR)
                xdp_do_flush_map();

        if (xdp_xmit & IXGBE_XDP_TX) {
                struct ixgbe_ring *ring = adapter->xdp_ring[smp_processor_id()];

                /* Force memory writes to complete before letting h/w
                 * know there are new descriptors to fetch.
                 */
                wmb();
                writel(ring->next_to_use, ring->tail);
        }

        u64_stats_update_begin(&rx_ring->syncp);
        rx_ring->stats.packets += total_rx_packets;
        rx_ring->stats.bytes += total_rx_bytes;
        u64_stats_update_end(&rx_ring->syncp);
        q_vector->rx.total_packets += total_rx_packets;
        q_vector->rx.total_bytes += total_rx_bytes;

        return total_rx_packets;
}

/**
 * ixgbe_configure_msix - Configure MSI-X hardware
 * @adapter: board private structure
 *
 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
 * interrupts.
 **/
static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
{
        struct ixgbe_q_vector *q_vector;
        int v_idx;
        u32 mask;

        /* Populate MSIX to EITR Select */
        if (adapter->num_vfs > 32) {
                u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
        }

        /*
         * Populate the IVAR table and set the ITR values to the
         * corresponding register.
         */
        for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
                struct ixgbe_ring *ring;
                q_vector = adapter->q_vector[v_idx];

                ixgbe_for_each_ring(ring, q_vector->rx)
                        ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);

                ixgbe_for_each_ring(ring, q_vector->tx)
                        ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);

                ixgbe_write_eitr(q_vector);
        }

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
                               v_idx);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                ixgbe_set_ivar(adapter, -1, 1, v_idx);
                break;
        default:
                break;
        }
        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);

        /* set up to autoclear timer, and the vectors */
        mask = IXGBE_EIMS_ENABLE_MASK;
        mask &= ~(IXGBE_EIMS_OTHER |
                  IXGBE_EIMS_MAILBOX |
                  IXGBE_EIMS_LSC);

        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
}

/**
 * ixgbe_update_itr - update the dynamic ITR value based on statistics
 * @q_vector: structure containing interrupt and ring information
 * @ring_container: structure containing ring performance data
 *
 *      Stores a new ITR value based on packets and byte
 *      counts during the last interrupt.  The advantage of per interrupt
 *      computation is faster updates and more accurate ITR for the current
 *      traffic pattern.  Constants in this function were computed
 *      based on theoretical maximum wire speed and thresholds were set based
 *      on testing data as well as attempting to minimize response time
 *      while increasing bulk throughput.
 **/
static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
                             struct ixgbe_ring_container *ring_container)
{
        unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS |
                           IXGBE_ITR_ADAPTIVE_LATENCY;
        unsigned int avg_wire_size, packets, bytes;
        unsigned long next_update = jiffies;

        /* If we don't have any rings just leave ourselves set for maximum
         * possible latency so we take ourselves out of the equation.
         */
        if (!ring_container->ring)
                return;

        /* If we didn't update within up to 1 - 2 jiffies we can assume
         * that either packets are coming in so slow there hasn't been
         * any work, or that there is so much work that NAPI is dealing
         * with interrupt moderation and we don't need to do anything.
         */
        if (time_after(next_update, ring_container->next_update))
                goto clear_counts;

        packets = ring_container->total_packets;

        /* We have no packets to actually measure against. This means
         * either one of the other queues on this vector is active or
         * we are a Tx queue doing TSO with too high of an interrupt rate.
         *
         * When this occurs just tick up our delay by the minimum value
         * and hope that this extra delay will prevent us from being called
         * without any work on our queue.
         */
        if (!packets) {
                itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
                if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
                itr += ring_container->itr & IXGBE_ITR_ADAPTIVE_LATENCY;
                goto clear_counts;
        }

        bytes = ring_container->total_bytes;

        /* If packets are less than 4 or bytes are less than 9000 assume
         * insufficient data to use bulk rate limiting approach. We are
         * likely latency driven.
         */
        if (packets < 4 && bytes < 9000) {
                itr = IXGBE_ITR_ADAPTIVE_LATENCY;
                goto adjust_by_size;
        }

        /* Between 4 and 48 we can assume that our current interrupt delay
         * is only slightly too low. As such we should increase it by a small
         * fixed amount.
         */
        if (packets < 48) {
                itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
                if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
                goto clear_counts;
        }

        /* Between 48 and 96 is our "goldilocks" zone where we are working
         * out "just right". Just report that our current ITR is good for us.
         */
        if (packets < 96) {
                itr = q_vector->itr >> 2;
                goto clear_counts;
        }

        /* If packet count is 96 or greater we are likely looking at a slight
         * overrun of the delay we want. Try halving our delay to see if that
         * will cut the number of packets in half per interrupt.
         */
        if (packets < 256) {
                itr = q_vector->itr >> 3;
                if (itr < IXGBE_ITR_ADAPTIVE_MIN_USECS)
                        itr = IXGBE_ITR_ADAPTIVE_MIN_USECS;
                goto clear_counts;
        }

        /* The paths below assume we are dealing with a bulk ITR since number
         * of packets is 256 or greater. We are just going to have to compute
         * a value and try to bring the count under control, though for smaller
         * packet sizes there isn't much we can do as NAPI polling will likely
         * be kicking in sooner rather than later.
         */
        itr = IXGBE_ITR_ADAPTIVE_BULK;

adjust_by_size:
        /* If packet counts are 256 or greater we can assume we have a gross
         * overestimation of what the rate should be. Instead of trying to fine
         * tune it just use the formula below to try and dial in an exact value
         * give the current packet size of the frame.
         */
        avg_wire_size = bytes / packets;

        /* The following is a crude approximation of:
         *  wmem_default / (size + overhead) = desired_pkts_per_int
         *  rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
         *  (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
         *
         * Assuming wmem_default is 212992 and overhead is 640 bytes per
         * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
         * formula down to
         *
         *  (170 * (size + 24)) / (size + 640) = ITR
         *
         * We first do some math on the packet size and then finally bitshift
         * by 8 after rounding up. We also have to account for PCIe link speed
         * difference as ITR scales based on this.
         */
        if (avg_wire_size <= 60) {
                /* Start at 50k ints/sec */
                avg_wire_size = 5120;
        } else if (avg_wire_size <= 316) {
                /* 50K ints/sec to 16K ints/sec */
                avg_wire_size *= 40;
                avg_wire_size += 2720;
        } else if (avg_wire_size <= 1084) {
                /* 16K ints/sec to 9.2K ints/sec */
                avg_wire_size *= 15;
                avg_wire_size += 11452;
        } else if (avg_wire_size < 1968) {
                /* 9.2K ints/sec to 8K ints/sec */
                avg_wire_size *= 5;
                avg_wire_size += 22420;
        } else {
                /* plateau at a limit of 8K ints/sec */
                avg_wire_size = 32256;
        }

        /* If we are in low latency mode half our delay which doubles the rate
         * to somewhere between 100K to 16K ints/sec
         */
        if (itr & IXGBE_ITR_ADAPTIVE_LATENCY)
                avg_wire_size >>= 1;

        /* Resultant value is 256 times larger than it needs to be. This
         * gives us room to adjust the value as needed to either increase
         * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
         *
         * Use addition as we have already recorded the new latency flag
         * for the ITR value.
         */
        switch (q_vector->adapter->link_speed) {
        case IXGBE_LINK_SPEED_10GB_FULL:
        case IXGBE_LINK_SPEED_100_FULL:
        default:
                itr += DIV_ROUND_UP(avg_wire_size,
                                    IXGBE_ITR_ADAPTIVE_MIN_INC * 256) *
                       IXGBE_ITR_ADAPTIVE_MIN_INC;
                break;
        case IXGBE_LINK_SPEED_2_5GB_FULL:
        case IXGBE_LINK_SPEED_1GB_FULL:
        case IXGBE_LINK_SPEED_10_FULL:
                if (avg_wire_size > 8064)
                        avg_wire_size = 8064;
                itr += DIV_ROUND_UP(avg_wire_size,
                                    IXGBE_ITR_ADAPTIVE_MIN_INC * 64) *
                       IXGBE_ITR_ADAPTIVE_MIN_INC;
                break;
        }

clear_counts:
        /* write back value */
        ring_container->itr = itr;

        /* next update should occur within next jiffy */
        ring_container->next_update = next_update + 1;

        ring_container->total_bytes = 0;
        ring_container->total_packets = 0;
}

/**
 * ixgbe_write_eitr - write EITR register in hardware specific way
 * @q_vector: structure containing interrupt and ring information
 *
 * This function is made to be called by ethtool and by the driver
 * when it needs to update EITR registers at runtime.  Hardware
 * specific quirks/differences are taken care of here.
 */
void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
{
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_hw *hw = &adapter->hw;
        int v_idx = q_vector->v_idx;
        u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                /* must write high and low 16 bits to reset counter */
                itr_reg |= (itr_reg << 16);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                /*
                 * set the WDIS bit to not clear the timer bits and cause an
                 * immediate assertion of the interrupt
                 */
                itr_reg |= IXGBE_EITR_CNT_WDIS;
                break;
        default:
                break;
        }
        IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
}

static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
{
        u32 new_itr;

        ixgbe_update_itr(q_vector, &q_vector->tx);
        ixgbe_update_itr(q_vector, &q_vector->rx);

        /* use the smallest value of new ITR delay calculations */
        new_itr = min(q_vector->rx.itr, q_vector->tx.itr);

        /* Clear latency flag if set, shift into correct position */
        new_itr &= ~IXGBE_ITR_ADAPTIVE_LATENCY;
        new_itr <<= 2;

        if (new_itr != q_vector->itr) {
                /* save the algorithm value here */
                q_vector->itr = new_itr;

                ixgbe_write_eitr(q_vector);
        }
}

/**
 * ixgbe_check_overtemp_subtask - check for over temperature
 * @adapter: pointer to adapter
 **/
static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr = adapter->interrupt_event;
        s32 rc;

        if (test_bit(__IXGBE_DOWN, &adapter->state))
                return;

        if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
                return;

        adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;

        switch (hw->device_id) {
        case IXGBE_DEV_ID_82599_T3_LOM:
                /*
                 * Since the warning interrupt is for both ports
                 * we don't have to check if:
                 *  - This interrupt wasn't for our port.
                 *  - We may have missed the interrupt so always have to
                 *    check if we  got a LSC
                 */
                if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
                    !(eicr & IXGBE_EICR_LSC))
                        return;

                if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
                        u32 speed;
                        bool link_up = false;

                        hw->mac.ops.check_link(hw, &speed, &link_up, false);

                        if (link_up)
                                return;
                }

                /* Check if this is not due to overtemp */
                if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
                        return;

                break;
        case IXGBE_DEV_ID_X550EM_A_1G_T:
        case IXGBE_DEV_ID_X550EM_A_1G_T_L:
                rc = hw->phy.ops.check_overtemp(hw);
                if (rc != IXGBE_ERR_OVERTEMP)
                        return;
                break;
        default:
                if (adapter->hw.mac.type >= ixgbe_mac_X540)
                        return;
                if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
                        return;
                break;
        }
        e_crit(drv, "%s\n", ixgbe_overheat_msg);

        adapter->interrupt_event = 0;
}

static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;

        if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
            (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
                e_crit(probe, "Fan has stopped, replace the adapter\n");
                /* write to clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
        }
}

static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;

        if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
                return;

        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
                /*
                 * Need to check link state so complete overtemp check
                 * on service task
                 */
                if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
                     (eicr & IXGBE_EICR_LSC)) &&
                    (!test_bit(__IXGBE_DOWN, &adapter->state))) {
                        adapter->interrupt_event = eicr;
                        adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
                        ixgbe_service_event_schedule(adapter);
                        return;
                }
                return;
        case ixgbe_mac_x550em_a:
                if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {
                        adapter->interrupt_event = eicr;
                        adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,
                                        IXGBE_EICR_GPI_SDP0_X550EM_a);
                        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,
                                        IXGBE_EICR_GPI_SDP0_X550EM_a);
                }
                return;
        case ixgbe_mac_X550:
        case ixgbe_mac_X540:
                if (!(eicr & IXGBE_EICR_TS))
                        return;
                break;
        default:
                return;
        }

        e_crit(drv, "%s\n", ixgbe_overheat_msg);
}

static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
{
        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                if (hw->phy.type == ixgbe_phy_nl)
                        return true;
                return false;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                switch (hw->mac.ops.get_media_type(hw)) {
                case ixgbe_media_type_fiber:
                case ixgbe_media_type_fiber_qsfp:
                        return true;
                default:
                        return false;
                }
        default:
                return false;
        }
}

static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);

        if (!ixgbe_is_sfp(hw))
                return;

        /* Later MAC's use different SDP */
        if (hw->mac.type >= ixgbe_mac_X540)
                eicr_mask = IXGBE_EICR_GPI_SDP0_X540;

        if (eicr & eicr_mask) {
                /* Clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
                if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                        adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
                        adapter->sfp_poll_time = 0;
                        ixgbe_service_event_schedule(adapter);
                }
        }

        if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
            (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
                /* Clear the interrupt */
                IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
                if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                        adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
                        ixgbe_service_event_schedule(adapter);
                }
        }
}

static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;

        adapter->lsc_int++;
        adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
        adapter->link_check_timeout = jiffies;
        if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
                IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
                IXGBE_WRITE_FLUSH(hw);
                ixgbe_service_event_schedule(adapter);
        }
}

static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
                                           u64 qmask)
{
        u32 mask;
        struct ixgbe_hw *hw = &adapter->hw;

        switch (hw->mac.type) {
        case ixgbe_mac_82598EB:
                mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
                IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                mask = (qmask & 0xFFFFFFFF);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
                mask = (qmask >> 32);
                if (mask)
                        IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
                break;
        default:
                break;
        }
        /* skip the flush */
}

/**
 * ixgbe_irq_enable - Enable default interrupt generation settings
 * @adapter: board private structure
 * @queues: enable irqs for queues
 * @flush: flush register write
 **/
static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
                                    bool flush)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);

        /* don't reenable LSC while waiting for link */
        if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
                mask &= ~IXGBE_EIMS_LSC;

        if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
                switch (adapter->hw.mac.type) {
                case ixgbe_mac_82599EB:
                        mask |= IXGBE_EIMS_GPI_SDP0(hw);
                        break;
                case ixgbe_mac_X540:

                case ixgbe_mac_X550:
                case ixgbe_mac_X550EM_x:
                case ixgbe_mac_x550em_a:
                        mask |= IXGBE_EIMS_TS;
                        break;
                default:
                        break;
                }
        if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
                mask |= IXGBE_EIMS_GPI_SDP1(hw);
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82599EB:
                mask |= IXGBE_EIMS_GPI_SDP1(hw);
                mask |= IXGBE_EIMS_GPI_SDP2(hw);
                fallthrough;
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
                    adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
                    adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
                        mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
                if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
                        mask |= IXGBE_EICR_GPI_SDP0_X540;
                mask |= IXGBE_EIMS_ECC;
                mask |= IXGBE_EIMS_MAILBOX;
                break;
        default:
                break;
        }

        if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
            !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
                mask |= IXGBE_EIMS_FLOW_DIR;

        IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
        if (queues)
                ixgbe_irq_enable_queues(adapter, ~0);
        if (flush)
                IXGBE_WRITE_FLUSH(&adapter->hw);
}

static irqreturn_t ixgbe_msix_other(int irq, void *data)
{
        struct ixgbe_adapter *adapter = data;
        struct ixgbe_hw *hw = &adapter->hw;
        u32 eicr;

        /*
         * Workaround for Silicon errata.  Use clear-by-write instead
         * of clear-by-read.  Reading with EICS will return the
         * interrupt causes without clearing, which later be done
         * with the write to EICR.
         */
        eicr = IXGBE_READ_REG(hw, IXGBE_EICS);

        /* The lower 16bits of the EICR register are for the queue interrupts
         * which should be masked here in order to not accidentally clear them if
         * the bits are high when ixgbe_msix_other is called. There is a race
         * condition otherwise which results in possible performance loss
         * especially if the ixgbe_msix_other interrupt is triggering
         * consistently (as it would when PPS is turned on for the X540 device)
         */
        eicr &= 0xFFFF0000;

        IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);

        if (eicr & IXGBE_EICR_LSC)
                ixgbe_check_lsc(adapter);

        if (eicr & IXGBE_EICR_MAILBOX)
                ixgbe_msg_task(adapter);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
                    (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
                        adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR,
                                        IXGBE_EICR_GPI_SDP0_X540);
                }
                if (eicr & IXGBE_EICR_ECC) {
                        e_info(link, "Received ECC Err, initiating reset\n");
                        set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
                }
                /* Handle Flow Director Full threshold interrupt */
                if (eicr & IXGBE_EICR_FLOW_DIR) {
                        int reinit_count = 0;
                        int i;
                        for (i = 0; i < adapter->num_tx_queues; i++) {
                                struct ixgbe_ring *ring = adapter->tx_ring[i];
                                if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
                                                       &ring->state))
                                        reinit_count++;
                        }
                        if (reinit_count) {
                                /* no more flow director interrupts until after init */
                                IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
                                adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
                                ixgbe_service_event_schedule(adapter);
                        }
                }
                ixgbe_check_sfp_event(adapter, eicr);
                ixgbe_check_overtemp_event(adapter, eicr);
                break;
        default:
                break;
        }

        ixgbe_check_fan_failure(adapter, eicr);

        if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
                ixgbe_ptp_check_pps_event(adapter);

        /* re-enable the original interrupt state, no lsc, no queues */
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
                ixgbe_irq_enable(adapter, false, false);

        return IRQ_HANDLED;
}

static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
{
        struct ixgbe_q_vector *q_vector = data;

        /* EIAM disabled interrupts (on this vector) for us */

        if (q_vector->rx.ring || q_vector->tx.ring)
                napi_schedule_irqoff(&q_vector->napi);

        return IRQ_HANDLED;
}

/**
 * ixgbe_poll - NAPI Rx polling callback
 * @napi: structure for representing this polling device
 * @budget: how many packets driver is allowed to clean
 *
 * This function is used for legacy and MSI, NAPI mode
 **/
int ixgbe_poll(struct napi_struct *napi, int budget)
{
        struct ixgbe_q_vector *q_vector =
                                container_of(napi, struct ixgbe_q_vector, napi);
        struct ixgbe_adapter *adapter = q_vector->adapter;
        struct ixgbe_ring *ring;
        int per_ring_budget, work_done = 0;
        bool clean_complete = true;

#ifdef CONFIG_IXGBE_DCA
        if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
                ixgbe_update_dca(q_vector);
#endif

        ixgbe_for_each_ring(ring, q_vector->tx) {
                bool wd = ring->xsk_pool ?
                          ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) :
                          ixgbe_clean_tx_irq(q_vector, ring, budget);

                if (!wd)
                        clean_complete = false;
        }

        /* Exit if we are called by netpoll */
        if (budget <= 0)
                return budget;

        /* attempt to distribute budget to each queue fairly, but don't allow
         * the budget to go below 1 because we'll exit polling */
        if (q_vector->rx.count > 1)
                per_ring_budget = max(budget/q_vector->rx.count, 1);
        else
                per_ring_budget = budget;

        ixgbe_for_each_ring(ring, q_vector->rx) {
                int cleaned = ring->xsk_pool ?
                              ixgbe_clean_rx_irq_zc(q_vector, ring,
                                                    per_ring_budget) :
                              ixgbe_clean_rx_irq(q_vector, ring,
                                                 per_ring_budget);

                work_done += cleaned;
                if (cleaned >= per_ring_budget)
                        clean_complete = false;
        }

        /* If all work not completed, return budget and keep polling */
        if (!clean_complete)
                return budget;

        /* all work done, exit the polling mode */
        if (likely(napi_complete_done(napi, work_done))) {
                if (adapter->rx_itr_setting & 1)
                        ixgbe_set_itr(q_vector);
                if (!test_bit(__IXGBE_DOWN, &adapter->state))
                        ixgbe_irq_enable_queues(adapter,
                                                BIT_ULL(q_vector->v_idx));
        }

        return min(work_done, budget - 1);
}

/**
 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
 * @adapter: board private structure
 *
 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
 * interrupts from the kernel.
 **/
static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        unsigned int ri = 0, ti = 0;
        int vector, err;

        for (vector = 0; vector < adapter->num_q_vectors; vector++) {
                struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
                struct msix_entry *entry = &adapter->msix_entries[vector];

                if (q_vector->tx.ring && q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-TxRx-%u", netdev->name, ri++);
                        ti++;
                } else if (q_vector->rx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-rx-%u", netdev->name, ri++);
                } else if (q_vector->tx.ring) {
                        snprintf(q_vector->name, sizeof(q_vector->name),
                                 "%s-tx-%u", netdev->name, ti++);
                } else {
                        /* skip this unused q_vector */
                        continue;
                }
                err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
                                  q_vector->name, q_vector);
                if (err) {
                        e_err(probe, "request_irq failed for MSIX interrupt "
                              "Error: %d\n", err);
                        goto free_queue_irqs;
                }
                /* If Flow Director is enabled, set interrupt affinity */
                if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
                        /* assign the mask for this irq */
                        irq_set_affinity_hint(entry->vector,
                                              &q_vector->affinity_mask);
                }
        }

        err = request_irq(adapter->msix_entries[vector].vector,
                          ixgbe_msix_other, 0, netdev->name, adapter);
        if (err) {
                e_err(probe, "request_irq for msix_other failed: %d\n", err);
                goto free_queue_irqs;
        }

        return 0;

free_queue_irqs:
        while (vector) {
                vector--;
                irq_set_affinity_hint(adapter->msix_entries[vector].vector,
                                      NULL);
                free_irq(adapter->msix_entries[vector].vector,
                         adapter->q_vector[vector]);
        }
        adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
        pci_disable_msix(adapter->pdev);
        kfree(adapter->msix_entries);
        adapter->msix_entries = NULL;
        return err;
}

/**
 * ixgbe_intr - legacy mode Interrupt Handler
 * @irq: interrupt number
 * @data: pointer to a network interface device structure
 **/
static irqreturn_t ixgbe_intr(int irq, void *data)
{
        struct ixgbe_adapter *adapter = data;
        struct ixgbe_hw *hw = &adapter->hw;
        struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
        u32 eicr;

        /*
         * Workaround for silicon errata #26 on 82598.  Mask the interrupt
         * before the read of EICR.
         */
        IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);

        /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
         * therefore no explicit interrupt disable is necessary */
        eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
        if (!eicr) {
                /*
                 * shared interrupt alert!
                 * make sure interrupts are enabled because the read will
                 * have disabled interrupts due to EIAM
                 * finish the workaround of silicon errata on 82598.  Unmask
                 * the interrupt that we masked before the EICR read.
                 */
                if (!test_bit(__IXGBE_DOWN, &adapter->state))
                        ixgbe_irq_enable(adapter, true, true);
                return IRQ_NONE;        /* Not our interrupt */
        }

        if (eicr & IXGBE_EICR_LSC)
                ixgbe_check_lsc(adapter);

        switch (hw->mac.type) {
        case ixgbe_mac_82599EB:
                ixgbe_check_sfp_event(adapter, eicr);
                fallthrough;
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                if (eicr & IXGBE_EICR_ECC) {
                        e_info(link, "Received ECC Err, initiating reset\n");
                        set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
                        ixgbe_service_event_schedule(adapter);
                        IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
                }
                ixgbe_check_overtemp_event(adapter, eicr);
                break;
        default:
                break;
        }

        ixgbe_check_fan_failure(adapter, eicr);
        if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
                ixgbe_ptp_check_pps_event(adapter);

        /* would disable interrupts here but EIAM disabled it */
        napi_schedule_irqoff(&q_vector->napi);

        /*
         * re-enable link(maybe) and non-queue interrupts, no flush.
         * ixgbe_poll will re-enable the queue interrupts
         */
        if (!test_bit(__IXGBE_DOWN, &adapter->state))
                ixgbe_irq_enable(adapter, false, false);

        return IRQ_HANDLED;
}

/**
 * ixgbe_request_irq - initialize interrupts
 * @adapter: board private structure
 *
 * Attempts to configure interrupts using the best available
 * capabilities of the hardware and kernel.
 **/
static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
{
        struct net_device *netdev = adapter->netdev;
        int err;

        if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
                err = ixgbe_request_msix_irqs(adapter);
        else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
                err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
                                  netdev->name, adapter);
        else
                err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
                                  netdev->name, adapter);

        if (err)
                e_err(probe, "request_irq failed, Error %d\n", err);

        return err;
}

static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
{
        int vector;

        if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
                free_irq(adapter->pdev->irq, adapter);
                return;
        }

        if (!adapter->msix_entries)
                return;

        for (vector = 0; vector < adapter->num_q_vectors; vector++) {
                struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
                struct msix_entry *entry = &adapter->msix_entries[vector];

                /* free only the irqs that were actually requested */
                if (!q_vector->rx.ring && !q_vector->tx.ring)
                        continue;

                /* clear the affinity_mask in the IRQ descriptor */
                irq_set_affinity_hint(entry->vector, NULL);

                free_irq(entry->vector, q_vector);
        }

        free_irq(adapter->msix_entries[vector].vector, adapter);
}

/**
 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
 * @adapter: board private structure
 **/
static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
{
        switch (adapter->hw.mac.type) {
        case ixgbe_mac_82598EB:
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
                break;
        case ixgbe_mac_82599EB:
        case ixgbe_mac_X540:
        case ixgbe_mac_X550:
        case ixgbe_mac_X550EM_x:
        case ixgbe_mac_x550em_a:
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
                IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
                break;
        default:
                break;
        }
        IXGBE_WRITE_FLUSH(&adapter->hw);
        if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
                int vector;

                for (vector = 0; vector < adapter->num_q_vectors; vector++)
                        synchronize_irq(adapter->msix_entries[vector].vector);

                synchronize_irq(adapter->msix_entries[vector++].vector);
        } else {
                synchronize_irq(adapter->pdev->irq);
        }
}

/**
 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
 * @adapter: board private structure
 *
 **/
static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
{
        struct ixgbe_q_vector *q_vector = adapter->q_vector[0];

        ixgbe_write_eitr(q_vector);

        ixgbe_set_ivar(adapter, 0, 0, 0);
        ixgbe_set_ivar(adapter, 1, 0, 0);

        e_info(hw, "Legacy interrupt IVAR setup done\n");
}

/**
 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
 * @adapter: board private structure
 * @ring: structure containing ring specific data
 *
 * Configure the Tx descriptor ring after a reset.
 **/
void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
                             struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u64 tdba = ring->dma;
        int wait_loop = 10;
        u32 txdctl = IXGBE_TXDCTL_ENABLE;
        u8 reg_idx = ring->reg_idx;

        ring->xsk_pool = NULL;
        if (ring_is_xdp(ring))
                ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);

        /* disable queue to avoid issues while updating state */
        IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
        IXGBE_WRITE_FLUSH(hw);

        IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
                        (tdba & DMA_BIT_MASK(32)));
        IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
        IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
                        ring->count * sizeof(union ixgbe_adv_tx_desc));
        IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
        IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
        ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);

        /*
         * set WTHRESH to encourage burst writeback, it should not be set
         * higher than 1 when:
         * - ITR is 0 as it could cause false TX hangs
         * - ITR is set to > 100k int/sec and BQL is enabled
         *
         * In order to avoid issues WTHRESH + PTHRESH should always be equal
         * to or less than the number of on chip descriptors, which is
         * currently 40.
         */
        if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
                txdctl |= 1u << 16;     /* WTHRESH = 1 */
        else
                txdctl |= 8u << 16;     /* WTHRESH = 8 */

        /*
         * Setting PTHRESH to 32 both improves performance
         * and avoids a TX hang with DFP enabled
         */
        txdctl |= (1u << 8) |   /* HTHRESH = 1 */
                   32;          /* PTHRESH = 32 */

        /* reinitialize flowdirector state */
        if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
                ring->atr_sample_rate = adapter->atr_sample_rate;
                ring->atr_count = 0;
                set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
        } else {
                ring->atr_sample_rate = 0;
        }

        /* initialize XPS */
        if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {
                struct ixgbe_q_vector *q_vector = ring->q_vector;

                if (q_vector)
                        netif_set_xps_queue(ring->netdev,
                                            &q_vector->affinity_mask,
                                            ring->queue_index);
        }

        clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);

        /* reinitialize tx_buffer_info */
        memset(ring->tx_buffer_info, 0,
               sizeof(struct ixgbe_tx_buffer) * ring->count);

        /* enable queue */
        IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);

        /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
        if (hw->mac.type == ixgbe_mac_82598EB &&
            !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
                return;

        /* poll to verify queue is enabled */
        do {
                usleep_range(1000, 2000);
                txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
        } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
        if (!wait_loop)
                hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
}

static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 rttdcs, mtqc;
        u8 tcs = adapter->hw_tcs;

        if (hw->mac.type == ixgbe_mac_82598EB)
                return;

        /* disable the arbiter while setting MTQC */
        rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
        rttdcs |= IXGBE_RTTDCS_ARBDIS;
        IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);

        /* set transmit pool layout */
        if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
                mtqc = IXGBE_MTQC_VT_ENA;
                if (tcs > 4)
                        mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
                else if (tcs > 1)
                        mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
                else if (adapter->ring_feature[RING_F_VMDQ].mask ==
                         IXGBE_82599_VMDQ_4Q_MASK)
                        mtqc |= IXGBE_MTQC_32VF;
                else
                        mtqc |= IXGBE_MTQC_64VF;
        } else {
                if (tcs > 4) {
                        mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
                } else if (tcs > 1) {
                        mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
                } else {
                        u8 max_txq = adapter->num_tx_queues +
                                adapter->num_xdp_queues;
                        if (max_txq > 63)
                                mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
                        else
                                mtqc = IXGBE_MTQC_64Q_1PB;
                }
        }

        IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);

        /* Enable Security TX Buffer IFG for multiple pb */
        if (tcs) {
                u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
                sectx |= IXGBE_SECTX_DCB;
                IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
        }

        /* re-enable the arbiter */
        rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
        IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
}

/**
 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
 * @adapter: board private structure
 *
 * Configure the Tx unit of the MAC after a reset.
 **/
static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 dmatxctl;
        u32 i;

        ixgbe_setup_mtqc(adapter);

        if (hw->mac.type != ixgbe_mac_82598EB) {
                /* DMATXCTL.EN must be before Tx queues are enabled */
                dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
                dmatxctl |= IXGBE_DMATXCTL_TE;
                IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
        }

        /* Setup the HW Tx Head and Tail descriptor pointers */
        for (i = 0; i < adapter->num_tx_queues; i++)
                ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
        for (i = 0; i < adapter->num_xdp_queues; i++)
                ixgbe_configure_tx_ring(adapter, adapter->xdp_ring[i]);
}

static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
                                 struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u8 reg_idx = ring->reg_idx;
        u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

        srrctl |= IXGBE_SRRCTL_DROP_EN;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
                                  struct ixgbe_ring *ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u8 reg_idx = ring->reg_idx;
        u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));

        srrctl &= ~IXGBE_SRRCTL_DROP_EN;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

#ifdef CONFIG_IXGBE_DCB
void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#else
static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
#endif
{
        int i;
        bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;

        if (adapter->ixgbe_ieee_pfc)
                pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);

        /*
         * We should set the drop enable bit if:
         *  SR-IOV is enabled
         *   or
         *  Number of Rx queues > 1 and flow control is disabled
         *
         *  This allows us to avoid head of line blocking for security
         *  and performance reasons.
         */
        if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
            !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
                for (i = 0; i < adapter->num_rx_queues; i++)
                        ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
        } else {
                for (i = 0; i < adapter->num_rx_queues; i++)
                        ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
        }
}

#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2

static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
                                   struct ixgbe_ring *rx_ring)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 srrctl;
        u8 reg_idx = rx_ring->reg_idx;

        if (hw->mac.type == ixgbe_mac_82598EB) {
                u16 mask = adapter->ring_feature[RING_F_RSS].mask;

                /*
                 * if VMDq is not active we must program one srrctl register
                 * per RSS queue since we have enabled RDRXCTL.MVMEN
                 */
                reg_idx &= mask;
        }

        /* configure header buffer length, needed for RSC */
        srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;

        /* configure the packet buffer length */
        if (rx_ring->xsk_pool) {
                u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool);

                /* If the MAC support setting RXDCTL.RLPML, the
                 * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and
                 * RXDCTL.RLPML is set to the actual UMEM buffer
                 * size. If not, then we are stuck with a 1k buffer
                 * size resolution. In this case frames larger than
                 * the UMEM buffer size viewed in a 1k resolution will
                 * be dropped.
                 */
                if (hw->mac.type != ixgbe_mac_82599EB)
                        srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
                else
                        srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
        } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {
                srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
        } else {
                srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
        }

        /* configure descriptor type */
        srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;

        IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
}

/**
 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
 * @adapter: device handle
 *
 *  - 82598/82599/X540:     128
 *  - X550(non-SRIOV mode): 512
 *  - X550(SRIOV mode):     64
 */
u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
{
        if (adapter->hw.mac.type < ixgbe_mac_X550)
                return 128;
        else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
                return 64;
        else
                return 512;
}

/**
 * ixgbe_store_key - Write the RSS key to HW
 * @adapter: device handle
 *
 * Write the RSS key stored in adapter.rss_key to HW.
 */
void ixgbe_store_key(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        int i;

        for (i = 0; i < 10; i++)
                IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
}

/**
 * ixgbe_init_rss_key - Initialize adapter RSS key
 * @adapter: device handle
 *
 * Allocates and initializes the RSS key if it is not allocated.
 **/
static inline int ixgbe_init_rss_key(struct ixgbe_adapter *adapter)
{
        u32 *rss_key;

        if (!adapter->rss_key) {
                rss_key = kzalloc(IXGBE_RSS_KEY_SIZE, GFP_KERNEL);
                if (unlikely(!rss_key))
                        return -ENOMEM;

                netdev_rss_key_fill(rss_key, IXGBE_RSS_KEY_SIZE);
                adapter->rss_key = rss_key;
        }

        return 0;
}

/**
 * ixgbe_store_reta - Write the RETA table to HW
 * @adapter: device handle
 *
 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
 */
void ixgbe_store_reta(struct ixgbe_adapter *adapter)
{
        u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 reta = 0;
        u32 indices_multi;
        u8 *indir_tbl = adapter->rss_indir_tbl;

        /* Fill out the redirection table as follows:
         *  - 82598:      8 bit wide entries containing pair of 4 bit RSS
         *    indices.
         *  - 82599/X540: 8 bit wide entries containing 4 bit RSS index
         *  - X550:       8 bit wide entries containing 6 bit RSS index
         */
        if (adapter->hw.mac.type == ixgbe_mac_82598EB)
                indices_multi = 0x11;
        else
                indices_multi = 0x1;

        /* Write redirection table to HW */
        for (i = 0; i < reta_entries; i++) {
                reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
                if ((i & 3) == 3) {
                        if (i < 128)
                                IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
                        else
                                IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
                                                reta);
                        reta = 0;
                }
        }
}

/**
 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
 * @adapter: device handle
 *
 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
 */
static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
{
        u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        struct ixgbe_hw *hw = &adapter->hw;
        u32 vfreta = 0;

        /* Write redirection table to HW */
        for (i = 0; i < reta_entries; i++) {
                u16 pool = adapter->num_rx_pools;

                vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
                if ((i & 3) != 3)
                        continue;

                while (pool--)
                        IXGBE_WRITE_REG(hw,
                                        IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),
                                        vfreta);
                vfreta = 0;
        }
}

static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
{
        u32 i, j;
        u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
        u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;

        /* Program table for at least 4 queues w/ SR-IOV so that VFs can
         * make full use of any rings they may have.  We will use the
         * PSRTYPE register to control how many rings we use within the PF.
         */
        if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
                rss_i = 4;

        /* Fill out hash function seeds */
        ixgbe_store_key(adapter);

        /* Fill out redirection table */
        memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));

        for (i = 0, j = 0; i < reta_entries; i++, j++) {
                if (j == rss_i)
                        j = 0;

                adapter->rss_indir_tbl[i] = j;
        }

        ixgbe_store_reta(adapter);
}

static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
        int i, j;

        /* Fill out hash function seeds */
        for (i = 0; i < 10; i++) {
                u16 pool = adapter->num_rx_pools;

                while (pool--)
                        IXGBE_WRITE_REG(hw,
                                        IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),
                                        *(adapter->rss_key + i));
        }

        /* Fill out the redirection table */
        for (i = 0, j = 0; i < 64; i++, j++) {
                if (j == rss_i)
                        j = 0;

                adapter->rss_indir_tbl[i] = j;
        }

        ixgbe_store_vfreta(adapter);
}

static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
{
        struct ixgbe_hw *hw = &adapter->hw;
        u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
        u32 rxcsum;

        /* Disable indicating checksum in descriptor, enables RSS hash */
        rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
        rxcsum |= IXGBE_RXCSUM_PCSD;
        IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);

        if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
                if (adapter->ring_feature[RING_F_RSS].mask)
                        mrqc = IXGBE_MRQC_RSSEN;
        } else {
                u8 tcs = adapter->hw_tcs;

                if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
                        if (tcs > 4)
                                mrqc = IXGBE_MRQC_VMDQRT8TCEN;  /* 8 TCs */
                        else if (tcs > 1)
                                mrqc = IXGBE_MRQC_VMDQRT4TCEN;  /* 4 TCs */
                        else if (adapter->ring_feature[RING_F_VMDQ].mask ==
                                 IXGBE_82599_VMDQ_4Q_MASK)
                                mrqc = IXGBE_MRQC_VMDQRSS32EN;
                        else
                                mrqc = IXGBE_MRQC_VMDQRSS64EN;

                        /* Enable L3/L4 for Tx Switched packets only for X550,
                         * older devices do not support this feature
                         */
                        if (hw->mac.type >= ixgbe_mac_X550)
                                mrqc |= IXGBE_MRQC_L3L4TXSWEN;
                } else {
                        if (tcs > 4)
                                mrqc = IXGBE_MRQC_RTRSS8TCEN;
                        else if (tcs > 1)
                                mrqc = IXGBE_MRQC_RTRSS4TCEN;
                        else
                                mrqc = IXGBE_MRQC_RSSEN;
                }
        }

        /* Perform hash on these packet types */
        rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
                     IXGBE_MRQC_RSS_FIELD_IPV6 |
                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP;

        if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
                rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
        if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
                rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;

        if ((hw->mac.type >= ixgbe_mac_X550) &&
            (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
                u16 pool = adapter->num_rx_pools;

                /* Enable VF RSS mode */
                mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
                IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);

                /* Setup RSS through the VF registers */
                ixgbe_setup_vfreta(adapter);
                vfmrqc = IXGBE_MRQC_RSSEN;
                vfmrqc |= rss_field;

                while (pool--)
                        IXGBE_WRITE_REG(hw,
                                        IXGBE_PFVFMRQC(VMDQ_P(pool)),
                                        vfmrqc);
        } else {
                ixgbe_setup_reta(adapter);
                mrqc |= rss_field;
                IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);