// SPDX-License-Identifier: GPL-2.0-only
/****************************************************************************
 * Driver for Solarflare network controllers and boards
 * Copyright 2005-2018 Solarflare Communications Inc.
 * Copyright 2019-2020 Xilinx Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation, incorporated herein by reference.
 */

#include "mcdi_filters.h"
#include "mcdi.h"
#include "nic.h"
#include "rx_common.h"

/* The maximum size of a shared RSS context */
/* TODO: this should really be from the mcdi protocol export */
#define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL

#define EFX_EF10_FILTER_ID_INVALID 0xffff

/* An arbitrary search limit for the software hash table */
#define EFX_EF10_FILTER_SEARCH_LIMIT 200

static struct efx_filter_spec *
efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table,
                           unsigned int filter_idx)
{
        return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
                                          ~EFX_EF10_FILTER_FLAGS);
}

static unsigned int
efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table,
                           unsigned int filter_idx)
{
        return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
}

static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id)
{
        WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID);
        return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1);
}

static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id)
{
        return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2);
}

static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx)
{
        return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx;
}

/*
 * Decide whether a filter should be exclusive or else should allow
 * delivery to additional recipients.  Currently we decide that
 * filters for specific local unicast MAC and IP addresses are
 * exclusive.
 */
static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec)
{
        if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
            !is_multicast_ether_addr(spec->loc_mac))
                return true;

        if ((spec->match_flags &
             (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
            (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
                if (spec->ether_type == htons(ETH_P_IP) &&
                    !ipv4_is_multicast(spec->loc_host[0]))
                        return true;
                if (spec->ether_type == htons(ETH_P_IPV6) &&
                    ((const u8 *)spec->loc_host)[0] != 0xff)
                        return true;
        }

        return false;
}

static void
efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table,
                          unsigned int filter_idx,
                          const struct efx_filter_spec *spec,
                          unsigned int flags)
{
        table->entry[filter_idx].spec = (unsigned long)spec | flags;
}

static void
efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx,
                                           const struct efx_filter_spec *spec,
                                           efx_dword_t *inbuf)
{
        enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
        u32 match_fields = 0, uc_match, mc_match;

        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
                       efx_mcdi_filter_is_exclusive(spec) ?
                       MC_CMD_FILTER_OP_IN_OP_INSERT :
                       MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);

        /*
         * Convert match flags and values.  Unlike almost
         * everything else in MCDI, these fields are in
         * network byte order.
         */
#define COPY_VALUE(value, mcdi_field)                                        \
        do {                                                         \
                match_fields |=                                      \
                        1 << MC_CMD_FILTER_OP_IN_MATCH_ ##           \
                        mcdi_field ## _LBN;                          \
                BUILD_BUG_ON(                                        \
                        MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
                        sizeof(value));                              \
                memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
                       &value, sizeof(value));                       \
        } while (0)
#define COPY_FIELD(gen_flag, gen_field, mcdi_field)                          \
        if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) {     \
                COPY_VALUE(spec->gen_field, mcdi_field);             \
        }
        /*
         * Handle encap filters first.  They will always be mismatch
         * (unknown UC or MC) filters
         */
        if (encap_type) {
                /*
                 * ether_type and outer_ip_proto need to be variables
                 * because COPY_VALUE wants to memcpy them
                 */
                __be16 ether_type =
                        htons(encap_type & EFX_ENCAP_FLAG_IPV6 ?
                              ETH_P_IPV6 : ETH_P_IP);
                u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE;
                u8 outer_ip_proto;

                switch (encap_type & EFX_ENCAP_TYPES_MASK) {
                case EFX_ENCAP_TYPE_VXLAN:
                        vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN;
                        fallthrough;
                case EFX_ENCAP_TYPE_GENEVE:
                        COPY_VALUE(ether_type, ETHER_TYPE);
                        outer_ip_proto = IPPROTO_UDP;
                        COPY_VALUE(outer_ip_proto, IP_PROTO);
                        /*
                         * We always need to set the type field, even
                         * though we're not matching on the TNI.
                         */
                        MCDI_POPULATE_DWORD_1(inbuf,
                                FILTER_OP_EXT_IN_VNI_OR_VSID,
                                FILTER_OP_EXT_IN_VNI_TYPE,
                                vni_type);
                        break;
                case EFX_ENCAP_TYPE_NVGRE:
                        COPY_VALUE(ether_type, ETHER_TYPE);
                        outer_ip_proto = IPPROTO_GRE;
                        COPY_VALUE(outer_ip_proto, IP_PROTO);
                        break;
                default:
                        WARN_ON(1);
                }

                uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
                mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
        } else {
                uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
                mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
        }

        if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
                match_fields |=
                        is_multicast_ether_addr(spec->loc_mac) ?
                        1 << mc_match :
                        1 << uc_match;
        COPY_FIELD(REM_HOST, rem_host, SRC_IP);
        COPY_FIELD(LOC_HOST, loc_host, DST_IP);
        COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
        COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
        COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
        COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
        COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
        COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
        COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
        COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
#undef COPY_FIELD
#undef COPY_VALUE
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
                       match_fields);
}

static void efx_mcdi_filter_push_prep(struct efx_nic *efx,
                                      const struct efx_filter_spec *spec,
                                      efx_dword_t *inbuf, u64 handle,
                                      struct efx_rss_context *ctx,
                                      bool replacing)
{
        u32 flags = spec->flags;

        memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN);

        /* If RSS filter, caller better have given us an RSS context */
        if (flags & EFX_FILTER_FLAG_RX_RSS) {
                /*
                 * We don't have the ability to return an error, so we'll just
                 * log a warning and disable RSS for the filter.
                 */
                if (WARN_ON_ONCE(!ctx))
                        flags &= ~EFX_FILTER_FLAG_RX_RSS;
                else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID))
                        flags &= ~EFX_FILTER_FLAG_RX_RSS;
        }

        if (replacing) {
                MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
                               MC_CMD_FILTER_OP_IN_OP_REPLACE);
                MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
        } else {
                efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf);
        }

        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id);
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
                       spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
                       MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
                       MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0);
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
                       MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE,
                       spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
                       0 : spec->dmaq_id);
        MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
                       (flags & EFX_FILTER_FLAG_RX_RSS) ?
                       MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
                       MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
        if (flags & EFX_FILTER_FLAG_RX_RSS)
                MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id);
}

static int efx_mcdi_filter_push(struct efx_nic *efx,
                                const struct efx_filter_spec *spec, u64 *handle,
                                struct efx_rss_context *ctx, bool replacing)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
        MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN);
        size_t outlen;
        int rc;

        efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing);
        rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
                                outbuf, sizeof(outbuf), &outlen);
        if (rc && spec->priority != EFX_FILTER_PRI_HINT)
                efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf),
                                       outbuf, outlen, rc);
        if (rc == 0)
                *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
        if (rc == -ENOSPC)
                rc = -EBUSY; /* to match efx_farch_filter_insert() */
        return rc;
}

static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
{
        enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
        unsigned int match_flags = spec->match_flags;
        unsigned int uc_match, mc_match;
        u32 mcdi_flags = 0;

#define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) {          \
                unsigned int  old_match_flags = match_flags;            \
                match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag;          \
                if (match_flags != old_match_flags)                     \
                        mcdi_flags |=                                   \
                                (1 << ((encap) ?                        \
                                       MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \
                                       mcdi_field ## _LBN :             \
                                       MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\
                                       mcdi_field ## _LBN));            \
        }
        /* inner or outer based on encap type */
        MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type);
        MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type);
        /* always outer */
        MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false);
        MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false);
#undef MAP_FILTER_TO_MCDI_FLAG

        /* special handling for encap type, and mismatch */
        if (encap_type) {
                match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE;
                mcdi_flags |=
                        (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
                mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);

                uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
                mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
        } else {
                uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
                mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
        }

        if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
                match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
                mcdi_flags |=
                        is_multicast_ether_addr(spec->loc_mac) ?
                        1 << mc_match :
                        1 << uc_match;
        }

        /* Did we map them all? */
        WARN_ON_ONCE(match_flags);

        return mcdi_flags;
}

static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table,
                               const struct efx_filter_spec *spec)
{
        u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
        unsigned int match_pri;

        for (match_pri = 0;
             match_pri < table->rx_match_count;
             match_pri++)
                if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
                        return match_pri;

        return -EPROTONOSUPPORT;
}

static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx,
                                         struct efx_filter_spec *spec,
                                         bool replace_equal)
{
        DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
        struct efx_mcdi_filter_table *table;
        struct efx_filter_spec *saved_spec;
        struct efx_rss_context *ctx = NULL;
        unsigned int match_pri, hash;
        unsigned int priv_flags;
        bool rss_locked = false;
        bool replacing = false;
        unsigned int depth, i;
        int ins_index = -1;
        DEFINE_WAIT(wait);
        bool is_mc_recip;
        s32 rc;

        WARN_ON(!rwsem_is_locked(&efx->filter_sem));
        table = efx->filter_state;
        down_write(&table->lock);

        /* For now, only support RX filters */
        if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
            EFX_FILTER_FLAG_RX) {
                rc = -EINVAL;
                goto out_unlock;
        }

        rc = efx_mcdi_filter_pri(table, spec);
        if (rc < 0)
                goto out_unlock;
        match_pri = rc;

        hash = efx_filter_spec_hash(spec);
        is_mc_recip = efx_filter_is_mc_recipient(spec);
        if (is_mc_recip)
                bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);

        if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
                mutex_lock(&efx->rss_lock);
                rss_locked = true;
                if (spec->rss_context)
                        ctx = efx_find_rss_context_entry(efx, spec->rss_context);
                else
                        ctx = &efx->rss_context;
                if (!ctx) {
                        rc = -ENOENT;
                        goto out_unlock;
                }
                if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
                        rc = -EOPNOTSUPP;
                        goto out_unlock;
                }
        }

        /* Find any existing filters with the same match tuple or
         * else a free slot to insert at.
         */
        for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
                i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
                saved_spec = efx_mcdi_filter_entry_spec(table, i);

                if (!saved_spec) {
                        if (ins_index < 0)
                                ins_index = i;
                } else if (efx_filter_spec_equal(spec, saved_spec)) {
                        if (spec->priority < saved_spec->priority &&
                            spec->priority != EFX_FILTER_PRI_AUTO) {
                                rc = -EPERM;
                                goto out_unlock;
                        }
                        if (!is_mc_recip) {
                                /* This is the only one */
                                if (spec->priority ==
                                    saved_spec->priority &&
                                    !replace_equal) {
                                        rc = -EEXIST;
                                        goto out_unlock;
                                }
                                ins_index = i;
                                break;
                        } else if (spec->priority >
                                   saved_spec->priority ||
                                   (spec->priority ==
                                    saved_spec->priority &&
                                    replace_equal)) {
                                if (ins_index < 0)
                                        ins_index = i;
                                else
                                        __set_bit(depth, mc_rem_map);
                        }
                }
        }

        /* Once we reach the maximum search depth, use the first suitable
         * slot, or return -EBUSY if there was none
         */
        if (ins_index < 0) {
                rc = -EBUSY;
                goto out_unlock;
        }

        /* Create a software table entry if necessary. */
        saved_spec = efx_mcdi_filter_entry_spec(table, ins_index);
        if (saved_spec) {
                if (spec->priority == EFX_FILTER_PRI_AUTO &&
                    saved_spec->priority >= EFX_FILTER_PRI_AUTO) {
                        /* Just make sure it won't be removed */
                        if (saved_spec->priority > EFX_FILTER_PRI_AUTO)
                                saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
                        table->entry[ins_index].spec &=
                                ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
                        rc = ins_index;
                        goto out_unlock;
                }
                replacing = true;
                priv_flags = efx_mcdi_filter_entry_flags(table, ins_index);
        } else {
                saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
                if (!saved_spec) {
                        rc = -ENOMEM;
                        goto out_unlock;
                }
                *saved_spec = *spec;
                priv_flags = 0;
        }
        efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);

        /* Actually insert the filter on the HW */
        rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle,
                                  ctx, replacing);

        if (rc == -EINVAL && efx->must_realloc_vis)
                /* The MC rebooted under us, causing it to reject our filter
                 * insertion as pointing to an invalid VI (spec->dmaq_id).
                 */
                rc = -EAGAIN;

        /* Finalise the software table entry */
        if (rc == 0) {
                if (replacing) {
                        /* Update the fields that may differ */
                        if (saved_spec->priority == EFX_FILTER_PRI_AUTO)
                                saved_spec->flags |=
                                        EFX_FILTER_FLAG_RX_OVER_AUTO;
                        saved_spec->priority = spec->priority;
                        saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO;
                        saved_spec->flags |= spec->flags;
                        saved_spec->rss_context = spec->rss_context;
                        saved_spec->dmaq_id = spec->dmaq_id;
                }
        } else if (!replacing) {
                kfree(saved_spec);
                saved_spec = NULL;
        } else {
                /* We failed to replace, so the old filter is still present.
                 * Roll back the software table to reflect this.  In fact the
                 * efx_mcdi_filter_set_entry() call below will do the right
                 * thing, so nothing extra is needed here.
                 */
        }
        efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);

        /* Remove and finalise entries for lower-priority multicast
         * recipients
         */
        if (is_mc_recip) {
                MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
                unsigned int depth, i;

                memset(inbuf, 0, sizeof(inbuf));

                for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
                        if (!test_bit(depth, mc_rem_map))
                                continue;

                        i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
                        saved_spec = efx_mcdi_filter_entry_spec(table, i);
                        priv_flags = efx_mcdi_filter_entry_flags(table, i);

                        if (rc == 0) {
                                MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
                                               MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
                                MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
                                               table->entry[i].handle);
                                rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
                                                  inbuf, sizeof(inbuf),
                                                  NULL, 0, NULL);
                        }

                        if (rc == 0) {
                                kfree(saved_spec);
                                saved_spec = NULL;
                                priv_flags = 0;
                        }
                        efx_mcdi_filter_set_entry(table, i, saved_spec,
                                                  priv_flags);
                }
        }

        /* If successful, return the inserted filter ID */
        if (rc == 0)
                rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index);

out_unlock:
        if (rss_locked)
                mutex_unlock(&efx->rss_lock);
        up_write(&table->lock);
        return rc;
}

s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec,
                           bool replace_equal)
{
        s32 ret;

        down_read(&efx->filter_sem);
        ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal);
        up_read(&efx->filter_sem);

        return ret;
}

/*
 * Remove a filter.
 * If !by_index, remove by ID
 * If by_index, remove by index
 * Filter ID may come from userland and must be range-checked.
 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
 * for write.
 */
static int efx_mcdi_filter_remove_internal(struct efx_nic *efx,
                                           unsigned int priority_mask,
                                           u32 filter_id, bool by_index)
{
        unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
        struct efx_mcdi_filter_table *table = efx->filter_state;
        MCDI_DECLARE_BUF(inbuf,
                         MC_CMD_FILTER_OP_IN_HANDLE_OFST +
                         MC_CMD_FILTER_OP_IN_HANDLE_LEN);
        struct efx_filter_spec *spec;
        DEFINE_WAIT(wait);
        int rc;

        spec = efx_mcdi_filter_entry_spec(table, filter_idx);
        if (!spec ||
            (!by_index &&
             efx_mcdi_filter_pri(table, spec) !=
             efx_mcdi_filter_get_unsafe_pri(filter_id)))
                return -ENOENT;

        if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
            priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
                /* Just remove flags */
                spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
                table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
                return 0;
        }

        if (!(priority_mask & (1U << spec->priority)))
                return -ENOENT;

        if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
                /* Reset to an automatic filter */

                struct efx_filter_spec new_spec = *spec;

                new_spec.priority = EFX_FILTER_PRI_AUTO;
                new_spec.flags = (EFX_FILTER_FLAG_RX |
                                  (efx_rss_active(&efx->rss_context) ?
                                   EFX_FILTER_FLAG_RX_RSS : 0));
                new_spec.dmaq_id = 0;
                new_spec.rss_context = 0;
                rc = efx_mcdi_filter_push(efx, &new_spec,
                                          &table->entry[filter_idx].handle,
                                          &efx->rss_context,
                                          true);

                if (rc == 0)
                        *spec = new_spec;
        } else {
                /* Really remove the filter */

                MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
                               efx_mcdi_filter_is_exclusive(spec) ?
                               MC_CMD_FILTER_OP_IN_OP_REMOVE :
                               MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
                MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
                               table->entry[filter_idx].handle);
                rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
                                        inbuf, sizeof(inbuf), NULL, 0, NULL);

                if ((rc == 0) || (rc == -ENOENT)) {
                        /* Filter removed OK or didn't actually exist */
                        kfree(spec);
                        efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
                } else {
                        efx_mcdi_display_error(efx, MC_CMD_FILTER_OP,
                                               MC_CMD_FILTER_OP_EXT_IN_LEN,
                                               NULL, 0, rc);
                }
        }

        return rc;
}

/* Remove filters that weren't renewed. */
static void efx_mcdi_filter_remove_old(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        int remove_failed = 0;
        int remove_noent = 0;
        int rc;
        int i;

        down_write(&table->lock);
        for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
                if (READ_ONCE(table->entry[i].spec) &
                    EFX_EF10_FILTER_FLAG_AUTO_OLD) {
                        rc = efx_mcdi_filter_remove_internal(efx,
                                        1U << EFX_FILTER_PRI_AUTO, i, true);
                        if (rc == -ENOENT)
                                remove_noent++;
                        else if (rc)
                                remove_failed++;
                }
        }
        up_write(&table->lock);

        if (remove_failed)
                netif_info(efx, drv, efx->net_dev,
                           "%s: failed to remove %d filters\n",
                           __func__, remove_failed);
        if (remove_noent)
                netif_info(efx, drv, efx->net_dev,
                           "%s: failed to remove %d non-existent filters\n",
                           __func__, remove_noent);
}

int efx_mcdi_filter_remove_safe(struct efx_nic *efx,
                                enum efx_filter_priority priority,
                                u32 filter_id)
{
        struct efx_mcdi_filter_table *table;
        int rc;

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_write(&table->lock);
        rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
                                             false);
        up_write(&table->lock);
        up_read(&efx->filter_sem);
        return rc;
}

/* Caller must hold efx->filter_sem for read */
static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx,
                                          enum efx_filter_priority priority,
                                          u32 filter_id)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;

        if (filter_id == EFX_EF10_FILTER_ID_INVALID)
                return;

        down_write(&table->lock);
        efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
                                        true);
        up_write(&table->lock);
}

int efx_mcdi_filter_get_safe(struct efx_nic *efx,
                             enum efx_filter_priority priority,
                             u32 filter_id, struct efx_filter_spec *spec)
{
        unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
        const struct efx_filter_spec *saved_spec;
        struct efx_mcdi_filter_table *table;
        int rc;

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_read(&table->lock);
        saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx);
        if (saved_spec && saved_spec->priority == priority &&
            efx_mcdi_filter_pri(table, saved_spec) ==
            efx_mcdi_filter_get_unsafe_pri(filter_id)) {
                *spec = *saved_spec;
                rc = 0;
        } else {
                rc = -ENOENT;
        }
        up_read(&table->lock);
        up_read(&efx->filter_sem);
        return rc;
}

static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx,
                                            struct efx_mcdi_filter_vlan *vlan,
                                            bool multicast, bool rollback)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_mcdi_dev_addr *addr_list;
        enum efx_filter_flags filter_flags;
        struct efx_filter_spec spec;
        u8 baddr[ETH_ALEN];
        unsigned int i, j;
        int addr_count;
        u16 *ids;
        int rc;

        if (multicast) {
                addr_list = table->dev_mc_list;
                addr_count = table->dev_mc_count;
                ids = vlan->mc;
        } else {
                addr_list = table->dev_uc_list;
                addr_count = table->dev_uc_count;
                ids = vlan->uc;
        }

        filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;

        /* Insert/renew filters */
        for (i = 0; i < addr_count; i++) {
                EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID);
                efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
                efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
                rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
                if (rc < 0) {
                        if (rollback) {
                                netif_info(efx, drv, efx->net_dev,
                                           "efx_mcdi_filter_insert failed rc=%d\n",
                                           rc);
                                /* Fall back to promiscuous */
                                for (j = 0; j < i; j++) {
                                        efx_mcdi_filter_remove_unsafe(
                                                efx, EFX_FILTER_PRI_AUTO,
                                                ids[j]);
                                        ids[j] = EFX_EF10_FILTER_ID_INVALID;
                                }
                                return rc;
                        } else {
                                /* keep invalid ID, and carry on */
                        }
                } else {
                        ids[i] = efx_mcdi_filter_get_unsafe_id(rc);
                }
        }

        if (multicast && rollback) {
                /* Also need an Ethernet broadcast filter */
                EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] !=
                                     EFX_EF10_FILTER_ID_INVALID);
                efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
                eth_broadcast_addr(baddr);
                efx_filter_set_eth_local(&spec, vlan->vid, baddr);
                rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
                if (rc < 0) {
                        netif_warn(efx, drv, efx->net_dev,
                                   "Broadcast filter insert failed rc=%d\n", rc);
                        /* Fall back to promiscuous */
                        for (j = 0; j < i; j++) {
                                efx_mcdi_filter_remove_unsafe(
                                        efx, EFX_FILTER_PRI_AUTO,
                                        ids[j]);
                                ids[j] = EFX_EF10_FILTER_ID_INVALID;
                        }
                        return rc;
                } else {
                        vlan->default_filters[EFX_EF10_BCAST] =
                                efx_mcdi_filter_get_unsafe_id(rc);
                }
        }

        return 0;
}

static int efx_mcdi_filter_insert_def(struct efx_nic *efx,
                                      struct efx_mcdi_filter_vlan *vlan,
                                      enum efx_encap_type encap_type,
                                      bool multicast, bool rollback)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        enum efx_filter_flags filter_flags;
        struct efx_filter_spec spec;
        u8 baddr[ETH_ALEN];
        int rc;
        u16 *id;

        filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;

        efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);

        if (multicast)
                efx_filter_set_mc_def(&spec);
        else
                efx_filter_set_uc_def(&spec);

        if (encap_type) {
                if (efx_has_cap(efx, VXLAN_NVGRE))
                        efx_filter_set_encap_type(&spec, encap_type);
                else
                        /*
                         * don't insert encap filters on non-supporting
                         * platforms. ID will be left as INVALID.
                         */
                        return 0;
        }

        if (vlan->vid != EFX_FILTER_VID_UNSPEC)
                efx_filter_set_eth_local(&spec, vlan->vid, NULL);

        rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
        if (rc < 0) {
                const char *um = multicast ? "Multicast" : "Unicast";
                const char *encap_name = "";
                const char *encap_ipv = "";

                if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
                    EFX_ENCAP_TYPE_VXLAN)
                        encap_name = "VXLAN ";
                else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
                         EFX_ENCAP_TYPE_NVGRE)
                        encap_name = "NVGRE ";
                else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
                         EFX_ENCAP_TYPE_GENEVE)
                        encap_name = "GENEVE ";
                if (encap_type & EFX_ENCAP_FLAG_IPV6)
                        encap_ipv = "IPv6 ";
                else if (encap_type)
                        encap_ipv = "IPv4 ";

                /*
                 * unprivileged functions can't insert mismatch filters
                 * for encapsulated or unicast traffic, so downgrade
                 * those warnings to debug.
                 */
                netif_cond_dbg(efx, drv, efx->net_dev,
                               rc == -EPERM && (encap_type || !multicast), warn,
                               "%s%s%s mismatch filter insert failed rc=%d\n",
                               encap_name, encap_ipv, um, rc);
        } else if (multicast) {
                /* mapping from encap types to default filter IDs (multicast) */
                static enum efx_mcdi_filter_default_filters map[] = {
                        [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF,
                        [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF,
                        [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF,
                        [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF,
                        [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_VXLAN6_MCDEF,
                        [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_NVGRE6_MCDEF,
                        [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_GENEVE6_MCDEF,
                };

                /* quick bounds check (BCAST result impossible) */
                BUILD_BUG_ON(EFX_EF10_BCAST != 0);
                if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
                        WARN_ON(1);
                        return -EINVAL;
                }
                /* then follow map */
                id = &vlan->default_filters[map[encap_type]];

                EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
                *id = efx_mcdi_filter_get_unsafe_id(rc);
                if (!table->mc_chaining && !encap_type) {
                        /* Also need an Ethernet broadcast filter */
                        efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
                                           filter_flags, 0);
                        eth_broadcast_addr(baddr);
                        efx_filter_set_eth_local(&spec, vlan->vid, baddr);
                        rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
                        if (rc < 0) {
                                netif_warn(efx, drv, efx->net_dev,
                                           "Broadcast filter insert failed rc=%d\n",
                                           rc);
                                if (rollback) {
                                        /* Roll back the mc_def filter */
                                        efx_mcdi_filter_remove_unsafe(
                                                        efx, EFX_FILTER_PRI_AUTO,
                                                        *id);
                                        *id = EFX_EF10_FILTER_ID_INVALID;
                                        return rc;
                                }
                        } else {
                                EFX_WARN_ON_PARANOID(
                                        vlan->default_filters[EFX_EF10_BCAST] !=
                                        EFX_EF10_FILTER_ID_INVALID);
                                vlan->default_filters[EFX_EF10_BCAST] =
                                        efx_mcdi_filter_get_unsafe_id(rc);
                        }
                }
                rc = 0;
        } else {
                /* mapping from encap types to default filter IDs (unicast) */
                static enum efx_mcdi_filter_default_filters map[] = {
                        [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF,
                        [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF,
                        [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF,
                        [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF,
                        [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_VXLAN6_UCDEF,
                        [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_NVGRE6_UCDEF,
                        [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
                                EFX_EF10_GENEVE6_UCDEF,
                };

                /* quick bounds check (BCAST result impossible) */
                BUILD_BUG_ON(EFX_EF10_BCAST != 0);
                if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
                        WARN_ON(1);
                        return -EINVAL;
                }
                /* then follow map */
                id = &vlan->default_filters[map[encap_type]];
                EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
                *id = rc;
                rc = 0;
        }
        return rc;
}

/*
 * Caller must hold efx->filter_sem for read if race against
 * efx_mcdi_filter_table_remove() is possible
 */
static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx,
                                              struct efx_mcdi_filter_vlan *vlan)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;

        /*
         * Do not install unspecified VID if VLAN filtering is enabled.
         * Do not install all specified VIDs if VLAN filtering is disabled.
         */
        if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
                return;

        /* Insert/renew unicast filters */
        if (table->uc_promisc) {
                efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE,
                                           false, false);
                efx_mcdi_filter_insert_addr_list(efx, vlan, false, false);
        } else {
                /*
                 * If any of the filters failed to insert, fall back to
                 * promiscuous mode - add in the uc_def filter.  But keep
                 * our individual unicast filters.
                 */
                if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false))
                        efx_mcdi_filter_insert_def(efx, vlan,
                                                   EFX_ENCAP_TYPE_NONE,
                                                   false, false);
        }
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,

                                   false, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
                                              EFX_ENCAP_FLAG_IPV6,
                                   false, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
                                   false, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
                                              EFX_ENCAP_FLAG_IPV6,
                                   false, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
                                   false, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
                                              EFX_ENCAP_FLAG_IPV6,
                                   false, false);

        /*
         * Insert/renew multicast filters
         *
         * If changing promiscuous state with cascaded multicast filters, remove
         * old filters first, so that packets are dropped rather than duplicated
         */
        if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc)
                efx_mcdi_filter_remove_old(efx);
        if (table->mc_promisc) {
                if (table->mc_chaining) {
                        /*
                         * If we failed to insert promiscuous filters, rollback
                         * and fall back to individual multicast filters
                         */
                        if (efx_mcdi_filter_insert_def(efx, vlan,
                                                       EFX_ENCAP_TYPE_NONE,
                                                       true, true)) {
                                /* Changing promisc state, so remove old filters */
                                efx_mcdi_filter_remove_old(efx);
                                efx_mcdi_filter_insert_addr_list(efx, vlan,
                                                                 true, false);
                        }
                } else {
                        /*
                         * If we failed to insert promiscuous filters, don't
                         * rollback.  Regardless, also insert the mc_list,
                         * unless it's incomplete due to overflow
                         */
                        efx_mcdi_filter_insert_def(efx, vlan,
                                                   EFX_ENCAP_TYPE_NONE,
                                                   true, false);
                        if (!table->mc_overflow)
                                efx_mcdi_filter_insert_addr_list(efx, vlan,
                                                                 true, false);
                }
        } else {
                /*
                 * If any filters failed to insert, rollback and fall back to
                 * promiscuous mode - mc_def filter and maybe broadcast.  If
                 * that fails, roll back again and insert as many of our
                 * individual multicast filters as we can.
                 */
                if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) {
                        /* Changing promisc state, so remove old filters */
                        if (table->mc_chaining)
                                efx_mcdi_filter_remove_old(efx);
                        if (efx_mcdi_filter_insert_def(efx, vlan,
                                                       EFX_ENCAP_TYPE_NONE,
                                                       true, true))
                                efx_mcdi_filter_insert_addr_list(efx, vlan,
                                                                 true, false);
                }
        }
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
                                   true, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
                                              EFX_ENCAP_FLAG_IPV6,
                                   true, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
                                   true, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
                                              EFX_ENCAP_FLAG_IPV6,
                                   true, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
                                   true, false);
        efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
                                              EFX_ENCAP_FLAG_IPV6,
                                   true, false);
}

int efx_mcdi_filter_clear_rx(struct efx_nic *efx,
                             enum efx_filter_priority priority)
{
        struct efx_mcdi_filter_table *table;
        unsigned int priority_mask;
        unsigned int i;
        int rc;

        priority_mask = (((1U << (priority + 1)) - 1) &
                         ~(1U << EFX_FILTER_PRI_AUTO));

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_write(&table->lock);
        for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
                rc = efx_mcdi_filter_remove_internal(efx, priority_mask,
                                                     i, true);
                if (rc && rc != -ENOENT)
                        break;
                rc = 0;
        }

        up_write(&table->lock);
        up_read(&efx->filter_sem);
        return rc;
}

u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx,
                                 enum efx_filter_priority priority)
{
        struct efx_mcdi_filter_table *table;
        unsigned int filter_idx;
        s32 count = 0;

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_read(&table->lock);
        for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
                if (table->entry[filter_idx].spec &&
                    efx_mcdi_filter_entry_spec(table, filter_idx)->priority ==
                    priority)
                        ++count;
        }
        up_read(&table->lock);
        up_read(&efx->filter_sem);
        return count;
}

u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;

        return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2;
}

s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx,
                               enum efx_filter_priority priority,
                               u32 *buf, u32 size)
{
        struct efx_mcdi_filter_table *table;
        struct efx_filter_spec *spec;
        unsigned int filter_idx;
        s32 count = 0;

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_read(&table->lock);

        for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
                spec = efx_mcdi_filter_entry_spec(table, filter_idx);
                if (spec && spec->priority == priority) {
                        if (count == size) {
                                count = -EMSGSIZE;
                                break;
                        }
                        buf[count++] =
                                efx_mcdi_filter_make_filter_id(
                                        efx_mcdi_filter_pri(table, spec),
                                        filter_idx);
                }
        }
        up_read(&table->lock);
        up_read(&efx->filter_sem);
        return count;
}

static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
{
        int match_flags = 0;

#define MAP_FLAG(gen_flag, mcdi_field) do {                             \
                u32 old_mcdi_flags = mcdi_flags;                        \
                mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##  \
                                     mcdi_field ## _LBN);               \
                if (mcdi_flags != old_mcdi_flags)                       \
                        match_flags |= EFX_FILTER_MATCH_ ## gen_flag;   \
        } while (0)

        if (encap) {
                /* encap filters must specify encap type */
                match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
                /* and imply ethertype and ip proto */
                mcdi_flags &=
                        ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
                mcdi_flags &=
                        ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
                /* VLAN tags refer to the outer packet */
                MAP_FLAG(INNER_VID, INNER_VLAN);
                MAP_FLAG(OUTER_VID, OUTER_VLAN);
                /* everything else refers to the inner packet */
                MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST);
                MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST);
                MAP_FLAG(REM_HOST, IFRM_SRC_IP);
                MAP_FLAG(LOC_HOST, IFRM_DST_IP);
                MAP_FLAG(REM_MAC, IFRM_SRC_MAC);
                MAP_FLAG(REM_PORT, IFRM_SRC_PORT);
                MAP_FLAG(LOC_MAC, IFRM_DST_MAC);
                MAP_FLAG(LOC_PORT, IFRM_DST_PORT);
                MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE);
                MAP_FLAG(IP_PROTO, IFRM_IP_PROTO);
        } else {
                MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
                MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
                MAP_FLAG(REM_HOST, SRC_IP);
                MAP_FLAG(LOC_HOST, DST_IP);
                MAP_FLAG(REM_MAC, SRC_MAC);
                MAP_FLAG(REM_PORT, SRC_PORT);
                MAP_FLAG(LOC_MAC, DST_MAC);
                MAP_FLAG(LOC_PORT, DST_PORT);
                MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
                MAP_FLAG(INNER_VID, INNER_VLAN);
                MAP_FLAG(OUTER_VID, OUTER_VLAN);
                MAP_FLAG(IP_PROTO, IP_PROTO);
        }
#undef MAP_FLAG

        /* Did we map them all? */
        if (mcdi_flags)
                return -EINVAL;

        return match_flags;
}

bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table,
                                     bool encap,
                                     enum efx_filter_match_flags match_flags)
{
        unsigned int match_pri;
        int mf;

        for (match_pri = 0;
             match_pri < table->rx_match_count;
             match_pri++) {
                mf = efx_mcdi_filter_match_flags_from_mcdi(encap,
                                table->rx_match_mcdi_flags[match_pri]);
                if (mf == match_flags)
                        return true;
        }

        return false;
}

static int
efx_mcdi_filter_table_probe_matches(struct efx_nic *efx,
                                    struct efx_mcdi_filter_table *table,
                                    bool encap)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
        MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
        unsigned int pd_match_pri, pd_match_count;
        size_t outlen;
        int rc;

        /* Find out which RX filter types are supported, and their priorities */
        MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
                       encap ?
                       MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
                       MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
        rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
                          inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
                          &outlen);
        if (rc)
                return rc;

        pd_match_count = MCDI_VAR_ARRAY_LEN(
                outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);

        for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
                u32 mcdi_flags =
                        MCDI_ARRAY_DWORD(
                                outbuf,
                                GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
                                pd_match_pri);
                rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags);
                if (rc < 0) {
                        netif_dbg(efx, probe, efx->net_dev,
                                  "%s: fw flags %#x pri %u not supported in driver\n",
                                  __func__, mcdi_flags, pd_match_pri);
                } else {
                        netif_dbg(efx, probe, efx->net_dev,
                                  "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
                                  __func__, mcdi_flags, pd_match_pri,
                                  rc, table->rx_match_count);
                        table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
                        table->rx_match_count++;
                }
        }

        return 0;
}

int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining)
{
        struct net_device *net_dev = efx->net_dev;
        struct efx_mcdi_filter_table *table;
        int rc;

        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return -EINVAL;

        if (efx->filter_state) /* already probed */
                return 0;

        table = kzalloc(sizeof(*table), GFP_KERNEL);
        if (!table)
                return -ENOMEM;

        table->mc_chaining = multicast_chaining;
        table->rx_match_count = 0;
        rc = efx_mcdi_filter_table_probe_matches(efx, table, false);
        if (rc)
                goto fail;
        if (efx_has_cap(efx, VXLAN_NVGRE))
                rc = efx_mcdi_filter_table_probe_matches(efx, table, true);
        if (rc)
                goto fail;
        if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
            !(efx_mcdi_filter_match_supported(table, false,
                (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
              efx_mcdi_filter_match_supported(table, false,
                (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
                netif_info(efx, probe, net_dev,
                           "VLAN filters are not supported in this firmware variant\n");
                net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
                efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
                net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
        }

        table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS,
                                          sizeof(*table->entry)));
        if (!table->entry) {
                rc = -ENOMEM;
                goto fail;
        }

        table->mc_promisc_last = false;
        table->vlan_filter =
                !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
        INIT_LIST_HEAD(&table->vlan_list);
        init_rwsem(&table->lock);

        efx->filter_state = table;

        return 0;
fail:
        kfree(table);
        return rc;
}

void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;

        if (table) {
                table->must_restore_filters = true;
                table->must_restore_rss_contexts = true;
        }
}

/*
 * Caller must hold efx->filter_sem for read if race against
 * efx_mcdi_filter_table_remove() is possible
 */
void efx_mcdi_filter_table_restore(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        unsigned int invalid_filters = 0, failed = 0;
        struct efx_mcdi_filter_vlan *vlan;
        struct efx_filter_spec *spec;
        struct efx_rss_context *ctx;
        unsigned int filter_idx;
        u32 mcdi_flags;
        int match_pri;
        int rc, i;

        WARN_ON(!rwsem_is_locked(&efx->filter_sem));

        if (!table || !table->must_restore_filters)
                return;

        down_write(&table->lock);
        mutex_lock(&efx->rss_lock);

        for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
                spec = efx_mcdi_filter_entry_spec(table, filter_idx);
                if (!spec)
                        continue;

                mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
                match_pri = 0;
                while (match_pri < table->rx_match_count &&
                       table->rx_match_mcdi_flags[match_pri] != mcdi_flags)
                        ++match_pri;
                if (match_pri >= table->rx_match_count) {
                        invalid_filters++;
                        goto not_restored;
                }
                if (spec->rss_context)
                        ctx = efx_find_rss_context_entry(efx, spec->rss_context);
                else
                        ctx = &efx->rss_context;
                if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
                        if (!ctx) {
                                netif_warn(efx, drv, efx->net_dev,
                                           "Warning: unable to restore a filter with nonexistent RSS context %u.\n",
                                           spec->rss_context);
                                invalid_filters++;
                                goto not_restored;
                        }
                        if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
                                netif_warn(efx, drv, efx->net_dev,
                                           "Warning: unable to restore a filter with RSS context %u as it was not created.\n",
                                           spec->rss_context);
                                invalid_filters++;
                                goto not_restored;
                        }
                }

                rc = efx_mcdi_filter_push(efx, spec,
                                          &table->entry[filter_idx].handle,
                                          ctx, false);
                if (rc)
                        failed++;

                if (rc) {
not_restored:
                        list_for_each_entry(vlan, &table->vlan_list, list)
                                for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i)
                                        if (vlan->default_filters[i] == filter_idx)
                                                vlan->default_filters[i] =
                                                        EFX_EF10_FILTER_ID_INVALID;

                        kfree(spec);
                        efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
                }
        }

        mutex_unlock(&efx->rss_lock);
        up_write(&table->lock);

        /*
         * This can happen validly if the MC's capabilities have changed, so
         * is not an error.
         */
        if (invalid_filters)
                netif_dbg(efx, drv, efx->net_dev,
                          "Did not restore %u filters that are now unsupported.\n",
                          invalid_filters);

        if (failed)
                netif_err(efx, hw, efx->net_dev,
                          "unable to restore %u filters\n", failed);
        else
                table->must_restore_filters = false;
}

void efx_mcdi_filter_table_down(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
        struct efx_filter_spec *spec;
        unsigned int filter_idx;
        int rc;

        if (!table)
                return;

        efx_mcdi_filter_cleanup_vlans(efx);

        for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
                spec = efx_mcdi_filter_entry_spec(table, filter_idx);
                if (!spec)
                        continue;

                MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
                               efx_mcdi_filter_is_exclusive(spec) ?
                               MC_CMD_FILTER_OP_IN_OP_REMOVE :
                               MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
                MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
                               table->entry[filter_idx].handle);
                rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf,
                                        sizeof(inbuf), NULL, 0, NULL);
                if (rc)
                        netif_info(efx, drv, efx->net_dev,
                                   "%s: filter %04x remove failed\n",
                                   __func__, filter_idx);
                kfree(spec);
        }
}

void efx_mcdi_filter_table_remove(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;

        efx_mcdi_filter_table_down(efx);

        efx->filter_state = NULL;
        /*
         * If we were called without locking, then it's not safe to free
         * the table as others might be using it.  So we just WARN, leak
         * the memory, and potentially get an inconsistent filter table
         * state.
         * This should never actually happen.
         */
        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return;

        if (!table)
                return;

        vfree(table->entry);
        kfree(table);
}

static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        unsigned int filter_idx;

        efx_rwsem_assert_write_locked(&table->lock);

        if (*id != EFX_EF10_FILTER_ID_INVALID) {
                filter_idx = efx_mcdi_filter_get_unsafe_id(*id);
                if (!table->entry[filter_idx].spec)
                        netif_dbg(efx, drv, efx->net_dev,
                                  "marked null spec old %04x:%04x\n", *id,
                                  filter_idx);
                table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
                *id = EFX_EF10_FILTER_ID_INVALID;
        }
}

/* Mark old per-VLAN filters that may need to be removed */
static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx,
                                           struct efx_mcdi_filter_vlan *vlan)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        unsigned int i;

        for (i = 0; i < table->dev_uc_count; i++)
                efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]);
        for (i = 0; i < table->dev_mc_count; i++)
                efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]);
        for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
                efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]);
}

/*
 * Mark old filters that may need to be removed.
 * Caller must hold efx->filter_sem for read if race against
 * efx_mcdi_filter_table_remove() is possible
 */
static void efx_mcdi_filter_mark_old(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_mcdi_filter_vlan *vlan;

        down_write(&table->lock);
        list_for_each_entry(vlan, &table->vlan_list, list)
                _efx_mcdi_filter_vlan_mark_old(efx, vlan);
        up_write(&table->lock);
}

int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_mcdi_filter_vlan *vlan;
        unsigned int i;

        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return -EINVAL;

        vlan = efx_mcdi_filter_find_vlan(efx, vid);
        if (WARN_ON(vlan)) {
                netif_err(efx, drv, efx->net_dev,
                          "VLAN %u already added\n", vid);
                return -EALREADY;
        }

        vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
        if (!vlan)
                return -ENOMEM;

        vlan->vid = vid;

        for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
                vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
        for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
                vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
        for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
                vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID;

        list_add_tail(&vlan->list, &table->vlan_list);

        if (efx_dev_registered(efx))
                efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);

        return 0;
}

static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx,
                                              struct efx_mcdi_filter_vlan *vlan)
{
        unsigned int i;

        /* See comment in efx_mcdi_filter_table_remove() */
        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return;

        list_del(&vlan->list);

        for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
                efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
                                              vlan->uc[i]);
        for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
                efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
                                              vlan->mc[i]);
        for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
                if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID)
                        efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
                                                      vlan->default_filters[i]);

        kfree(vlan);
}

void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid)
{
        struct efx_mcdi_filter_vlan *vlan;

        /* See comment in efx_mcdi_filter_table_remove() */
        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return;

        vlan = efx_mcdi_filter_find_vlan(efx, vid);
        if (!vlan) {
                netif_err(efx, drv, efx->net_dev,
                          "VLAN %u not found in filter state\n", vid);
                return;
        }

        efx_mcdi_filter_del_vlan_internal(efx, vlan);
}

struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx,
                                                       u16 vid)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_mcdi_filter_vlan *vlan;

        WARN_ON(!rwsem_is_locked(&efx->filter_sem));

        list_for_each_entry(vlan, &table->vlan_list, list) {
                if (vlan->vid == vid)
                        return vlan;
        }

        return NULL;
}

void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_mcdi_filter_vlan *vlan, *next_vlan;

        /* See comment in efx_mcdi_filter_table_remove() */
        if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
                return;

        if (!table)
                return;

        list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
                efx_mcdi_filter_del_vlan_internal(efx, vlan);
}

static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct net_device *net_dev = efx->net_dev;
        struct netdev_hw_addr *uc;
        unsigned int i;

        table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
        ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
        i = 1;
        netdev_for_each_uc_addr(uc, net_dev) {
                if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
                        table->uc_promisc = true;
                        break;
                }
                ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
                i++;
        }

        table->dev_uc_count = i;
}

static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct net_device *net_dev = efx->net_dev;
        struct netdev_hw_addr *mc;
        unsigned int i;

        table->mc_overflow = false;
        table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));

        i = 0;
        netdev_for_each_mc_addr(mc, net_dev) {
                if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
                        table->mc_promisc = true;
                        table->mc_overflow = true;
                        break;
                }
                ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
                i++;
        }

        table->dev_mc_count = i;
}

/*
 * Caller must hold efx->filter_sem for read if race against
 * efx_mcdi_filter_table_remove() is possible
 */
void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct net_device *net_dev = efx->net_dev;
        struct efx_mcdi_filter_vlan *vlan;
        bool vlan_filter;

        if (!efx_dev_registered(efx))
                return;

        if (!table)
                return;

        efx_mcdi_filter_mark_old(efx);

        /*
         * Copy/convert the address lists; add the primary station
         * address and broadcast address
         */
        netif_addr_lock_bh(net_dev);
        efx_mcdi_filter_uc_addr_list(efx);
        efx_mcdi_filter_mc_addr_list(efx);
        netif_addr_unlock_bh(net_dev);

        /*
         * If VLAN filtering changes, all old filters are finally removed.
         * Do it in advance to avoid conflicts for unicast untagged and
         * VLAN 0 tagged filters.
         */
        vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
        if (table->vlan_filter != vlan_filter) {
                table->vlan_filter = vlan_filter;
                efx_mcdi_filter_remove_old(efx);
        }

        list_for_each_entry(vlan, &table->vlan_list, list)
                efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);

        efx_mcdi_filter_remove_old(efx);
        table->mc_promisc_last = table->mc_promisc;
}

#ifdef CONFIG_RFS_ACCEL

bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
                                    unsigned int filter_idx)
{
        struct efx_filter_spec *spec, saved_spec;
        struct efx_mcdi_filter_table *table;
        struct efx_arfs_rule *rule = NULL;
        bool ret = true, force = false;
        u16 arfs_id;

        down_read(&efx->filter_sem);
        table = efx->filter_state;
        down_write(&table->lock);
        spec = efx_mcdi_filter_entry_spec(table, filter_idx);

        if (!spec || spec->priority != EFX_FILTER_PRI_HINT)
                goto out_unlock;

        spin_lock_bh(&efx->rps_hash_lock);
        if (!efx->rps_hash_table) {
                /* In the absence of the table, we always return 0 to ARFS. */
                arfs_id = 0;
        } else {
                rule = efx_rps_hash_find(efx, spec);
                if (!rule)
                        /* ARFS table doesn't know of this filter, so remove it */
                        goto expire;
                arfs_id = rule->arfs_id;
                ret = efx_rps_check_rule(rule, filter_idx, &force);
                if (force)
                        goto expire;
                if (!ret) {
                        spin_unlock_bh(&efx->rps_hash_lock);
                        goto out_unlock;
                }
        }
        if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))
                ret = false;
        else if (rule)
                rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
expire:
        saved_spec = *spec; /* remove operation will kfree spec */
        spin_unlock_bh(&efx->rps_hash_lock);
        /*
         * At this point (since we dropped the lock), another thread might queue
         * up a fresh insertion request (but the actual insertion will be held
         * up by our possession of the filter table lock).  In that case, it
         * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that
         * the rule is not removed by efx_rps_hash_del() below.
         */
        if (ret)
                ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority,
                                                      filter_idx, true) == 0;
        /*
         * While we can't safely dereference rule (we dropped the lock), we can
         * still test it for NULL.
         */
        if (ret && rule) {
                /* Expiring, so remove entry from ARFS table */
                spin_lock_bh(&efx->rps_hash_lock);
                efx_rps_hash_del(efx, &saved_spec);
                spin_unlock_bh(&efx->rps_hash_lock);
        }
out_unlock:
        up_write(&table->lock);
        up_read(&efx->filter_sem);
        return ret;
}

#endif /* CONFIG_RFS_ACCEL */

#define RSS_MODE_HASH_ADDRS     (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\
                                 1 << RSS_MODE_HASH_DST_ADDR_LBN)
#define RSS_MODE_HASH_PORTS     (1 << RSS_MODE_HASH_SRC_PORT_LBN |\
                                 1 << RSS_MODE_HASH_DST_PORT_LBN)
#define RSS_CONTEXT_FLAGS_DEFAULT       (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\
                                         1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\
                                         1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\
                                         1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\
                                         (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\
                                         RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\
                                         RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\
                                         (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\
                                         RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\
                                         RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN)

int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags)
{
        /*
         * Firmware had a bug (sfc bug 61952) where it would not actually
         * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS.
         * This meant that it would always contain whatever was previously
         * in the MCDI buffer.  Fortunately, all firmware versions with
         * this bug have the same default flags value for a newly-allocated
         * RSS context, and the only time we want to get the flags is just
         * after allocating.  Moreover, the response has a 32-bit hole
         * where the context ID would be in the request, so we can use an
         * overlength buffer in the request and pre-fill the flags field
         * with what we believe the default to be.  Thus if the firmware
         * has the bug, it will leave our pre-filled value in the flags
         * field of the response, and we will get the right answer.
         *
         * However, this does mean that this function should NOT be used if
         * the RSS context flags might not be their defaults - it is ONLY
         * reliably correct for a newly-allocated RSS context.
         */
        MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
        MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
        size_t outlen;
        int rc;

        /* Check we have a hole for the context ID */
        BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST);
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context);
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS,
                       RSS_CONTEXT_FLAGS_DEFAULT);
        rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf,
                          sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
        if (rc == 0) {
                if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN)
                        rc = -EIO;
                else
                        *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS);
        }
        return rc;
}

/*
 * Attempt to enable 4-tuple UDP hashing on the specified RSS context.
 * If we fail, we just leave the RSS context at its default hash settings,
 * which is safe but may slightly reduce performance.
 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we
 * just need to set the UDP ports flags (for both IP versions).
 */
void efx_mcdi_set_rss_context_flags(struct efx_nic *efx,
                                    struct efx_rss_context *ctx)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN);
        u32 flags;

        BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0);

        if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0)
                return;
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
                       ctx->context_id);
        flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN;
        flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN;
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags);
        if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf),
                          NULL, 0, NULL))
                /* Succeeded, so UDP 4-tuple is now enabled */
                ctx->rx_hash_udp_4tuple = true;
}

static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive,
                                             struct efx_rss_context *ctx,
                                             unsigned *context_size)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
        MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
        size_t outlen;
        int rc;
        u32 alloc_type = exclusive ?
                                MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE :
                                MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
        unsigned rss_spread = exclusive ?
                                efx->rss_spread :
                                min(rounddown_pow_of_two(efx->rss_spread),
                                    EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE);

        if (!exclusive && rss_spread == 1) {
                ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
                if (context_size)
                        *context_size = 1;
                return 0;
        }

        if (efx_has_cap(efx, RX_RSS_LIMITED))
                return -EOPNOTSUPP;

        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
                       efx->vport_id);
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type);
        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread);

        rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
                outbuf, sizeof(outbuf), &outlen);
        if (rc != 0)
                return rc;

        if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
                return -EIO;

        ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);

        if (context_size)
                *context_size = rss_spread;

        if (efx_has_cap(efx, ADDITIONAL_RSS_MODES))
                efx_mcdi_set_rss_context_flags(efx, ctx);

        return 0;
}

static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);

        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
                       context);
        return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
                            NULL, 0, NULL);
}

static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context,
                                       const u32 *rx_indir_table, const u8 *key)
{
        MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
        MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
        int i, rc;

        MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
                       context);
        BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
                     MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);

        /* This iterates over the length of efx->rss_context.rx_indir_table, but
         * copies bytes from rx_indir_table.  That's because the latter is a
         * pointer rather than an array, but should have the same length.
         * The efx->rss_context.rx_hash_key loop below is similar.
         */
        for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i)
                MCDI_PTR(tablebuf,
                         RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
                                (u8) rx_indir_table[i];

        rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
                          sizeof(tablebuf), NULL, 0, NULL);
        if (rc != 0)
                return rc;

        MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
                       context);
        BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) !=
                     MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
        for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i)
                MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i];

        return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
                            sizeof(keybuf), NULL, 0, NULL);
}

void efx_mcdi_rx_free_indir_table(struct efx_nic *efx)
{
        int rc;

        if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) {
                rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id);
                WARN_ON(rc != 0);
        }
        efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
}

static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx,
                                              unsigned *context_size)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context,
                                            context_size);

        if (rc != 0)
                return rc;

        table->rx_rss_context_exclusive = false;
        efx_set_default_rx_indir_table(efx, &efx->rss_context);
        return 0;
}

static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx,
                                                 const u32 *rx_indir_table,
                                                 const u8 *key)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        u32 old_rx_rss_context = efx->rss_context.context_id;
        int rc;

        if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID ||
            !table->rx_rss_context_exclusive) {
                rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context,
                                                NULL);
                if (rc == -EOPNOTSUPP)
                        return rc;
                else if (rc != 0)
                        goto fail1;
        }

        rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id,
                                         rx_indir_table, key);
        if (rc != 0)
                goto fail2;

        if (efx->rss_context.context_id != old_rx_rss_context &&
            old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID)
                WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0);
        table->rx_rss_context_exclusive = true;
        if (rx_indir_table != efx->rss_context.rx_indir_table)
                memcpy(efx->rss_context.rx_indir_table, rx_indir_table,
                       sizeof(efx->rss_context.rx_indir_table));
        if (key != efx->rss_context.rx_hash_key)
                memcpy(efx->rss_context.rx_hash_key, key,
                       efx->type->rx_hash_key_size);

        return 0;

fail2:
        if (old_rx_rss_context != efx->rss_context.context_id) {
                WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0);
                efx->rss_context.context_id = old_rx_rss_context;
        }
fail1:
        netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
        return rc;
}

int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx,
                                        struct efx_rss_context *ctx,
                                        const u32 *rx_indir_table,
                                        const u8 *key)
{
        int rc;

        WARN_ON(!mutex_is_locked(&efx->rss_lock));

        if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
                rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL);
                if (rc)
                        return rc;
        }

        if (!rx_indir_table) /* Delete this context */
                return efx_mcdi_filter_free_rss_context(efx, ctx->context_id);

        rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id,
                                         rx_indir_table, key);
        if (rc)
                return rc;

        memcpy(ctx->rx_indir_table, rx_indir_table,
               sizeof(efx->rss_context.rx_indir_table));
        memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size);

        return 0;
}

int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx,
                                        struct efx_rss_context *ctx)
{
        MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN);
        MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN);
        MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN);
        size_t outlen;
        int rc, i;

        WARN_ON(!mutex_is_locked(&efx->rss_lock));

        BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
                     MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);

        if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)
                return -ENOENT;

        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID,
                       ctx->context_id);
        BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) !=
                     MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN);
        rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf),
                          tablebuf, sizeof(tablebuf), &outlen);
        if (rc != 0)
                return rc;

        if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN))
                return -EIO;

        for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
                ctx->rx_indir_table[i] = MCDI_PTR(tablebuf,
                                RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i];

        MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID,
                       ctx->context_id);
        BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) !=
                     MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
        rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf),
                          keybuf, sizeof(keybuf), &outlen);
        if (rc != 0)
                return rc;

        if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN))
                return -EIO;

        for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i)
                ctx->rx_hash_key[i] = MCDI_PTR(
                                keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i];

        return 0;
}

int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx)
{
        int rc;

        mutex_lock(&efx->rss_lock);
        rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context);
        mutex_unlock(&efx->rss_lock);
        return rc;
}

void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx)
{
        struct efx_mcdi_filter_table *table = efx->filter_state;
        struct efx_rss_context *ctx;
        int rc;

        WARN_ON(!mutex_is_locked(&efx->rss_lock));

        if (!table->must_restore_rss_contexts)
                return;

        list_for_each_entry(ctx, &efx->rss_context.list, list) {
                /* previous NIC RSS context is gone */
                ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
                /* so try to allocate a new one */
                rc = efx_mcdi_rx_push_rss_context_config(efx, ctx,
                                                         ctx->rx_indir_table,
                                                         ctx->rx_hash_key);
                if (rc)
                        netif_warn(efx, probe, efx->net_dev,
                                   "failed to restore RSS context %u, rc=%d"
                                   "; RSS filters may fail to be applied\n",
                                   ctx->user_id, rc);
        }
        table->must_restore_rss_contexts = false;
}

int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
                                   const u32 *rx_indir_table,
                                   const u8 *key)
{
        int rc;

        if (efx->rss_spread == 1)
                return 0;

        if (!key)
                key = efx->rss_context.rx_hash_key;

        rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key);

        if (rc == -ENOBUFS && !user) {
                unsigned context_size;
                bool mismatch = false;
                size_t i;

                for (i = 0;
                     i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch;
                     i++)
                        mismatch = rx_indir_table[i] !=
                                ethtool_rxfh_indir_default(i, efx->rss_spread);

                rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size);
                if (rc == 0) {
                        if (context_size != efx->rss_spread)
                                netif_warn(efx, probe, efx->net_dev,
                                           "Could not allocate an exclusive RSS"
                                           " context; allocated a shared one of"
                                           " different size."
                                           " Wanted %u, got %u.\n",
                                           efx->rss_spread, context_size);
                        else if (mismatch)
                                netif_warn(efx, probe, efx->net_dev,
                                           "Could not allocate an exclusive RSS"
                                           " context; allocated a shared one but"
                                           " could not apply custom"
                                           " indirection.\n");
                        else
                                netif_info(efx, probe, efx->net_dev,
                                           "Could not allocate an exclusive RSS"
                                           " context; allocated a shared one.\n");
                }
        }
        return rc;
}

int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user,
                                   const u32 *rx_indir_table
                                   __attribute__ ((unused)),
                                   const u8 *key
                                   __attribute__ ((unused)))
{
        if (user)
                return -EOPNOTSUPP;
        if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID)
                return 0;
        return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL);
}

int efx_mcdi_push_default_indir_table(struct efx_nic *efx,
                                      unsigned int rss_spread)
{
        int rc = 0;

        if (efx->rss_spread == rss_spread)
                return 0;

        efx->rss_spread = rss_spread;
        if (!efx->filter_state)
                return 0;

        efx_mcdi_rx_free_indir_table(efx);
        if (rss_spread > 1) {
                efx_set_default_rx_indir_table(efx, &efx->rss_context);
                rc = efx->type->rx_push_rss_config(efx, false,
                                   efx->rss_context.rx_indir_table, NULL);
        }
        return rc;
}