// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2017-2018 Netronome Systems, Inc. */

#include <linux/etherdevice.h>
#include <linux/inetdevice.h>
#include <net/netevent.h>
#include <linux/idr.h>
#include <net/dst_metadata.h>
#include <net/arp.h>

#include "cmsg.h"
#include "main.h"
#include "../nfp_net_repr.h"
#include "../nfp_net.h"

#define NFP_FL_MAX_ROUTES               32

#define NFP_TUN_PRE_TUN_RULE_LIMIT      32
#define NFP_TUN_PRE_TUN_RULE_DEL        0x1
#define NFP_TUN_PRE_TUN_IDX_BIT         0x8

/**
 * struct nfp_tun_pre_run_rule - rule matched before decap
 * @flags:              options for the rule offset
 * @port_idx:           index of destination MAC address for the rule
 * @vlan_tci:           VLAN info associated with MAC
 * @host_ctx_id:        stats context of rule to update
 */
struct nfp_tun_pre_tun_rule {
        __be32 flags;
        __be16 port_idx;
        __be16 vlan_tci;
        __be32 host_ctx_id;
};

/**
 * struct nfp_tun_active_tuns - periodic message of active tunnels
 * @seq:                sequence number of the message
 * @count:              number of tunnels report in message
 * @flags:              options part of the request
 * @tun_info.ipv4:              dest IPv4 address of active route
 * @tun_info.egress_port:       port the encapsulated packet egressed
 * @tun_info.extra:             reserved for future use
 * @tun_info:           tunnels that have sent traffic in reported period
 */
struct nfp_tun_active_tuns {
        __be32 seq;
        __be32 count;
        __be32 flags;
        struct route_ip_info {
                __be32 ipv4;
                __be32 egress_port;
                __be32 extra[2];
        } tun_info[];
};

/**
 * struct nfp_tun_active_tuns_v6 - periodic message of active IPv6 tunnels
 * @seq:                sequence number of the message
 * @count:              number of tunnels report in message
 * @flags:              options part of the request
 * @tun_info.ipv6:              dest IPv6 address of active route
 * @tun_info.egress_port:       port the encapsulated packet egressed
 * @tun_info.extra:             reserved for future use
 * @tun_info:           tunnels that have sent traffic in reported period
 */
struct nfp_tun_active_tuns_v6 {
        __be32 seq;
        __be32 count;
        __be32 flags;
        struct route_ip_info_v6 {
                struct in6_addr ipv6;
                __be32 egress_port;
                __be32 extra[2];
        } tun_info[];
};

/**
 * struct nfp_tun_neigh - neighbour/route entry on the NFP
 * @dst_ipv4:   destination IPv4 address
 * @src_ipv4:   source IPv4 address
 * @dst_addr:   destination MAC address
 * @src_addr:   source MAC address
 * @port_id:    NFP port to output packet on - associated with source IPv4
 */
struct nfp_tun_neigh {
        __be32 dst_ipv4;
        __be32 src_ipv4;
        u8 dst_addr[ETH_ALEN];
        u8 src_addr[ETH_ALEN];
        __be32 port_id;
};

/**
 * struct nfp_tun_neigh_v6 - neighbour/route entry on the NFP
 * @dst_ipv6:   destination IPv6 address
 * @src_ipv6:   source IPv6 address
 * @dst_addr:   destination MAC address
 * @src_addr:   source MAC address
 * @port_id:    NFP port to output packet on - associated with source IPv6
 */
struct nfp_tun_neigh_v6 {
        struct in6_addr dst_ipv6;
        struct in6_addr src_ipv6;
        u8 dst_addr[ETH_ALEN];
        u8 src_addr[ETH_ALEN];
        __be32 port_id;
};

/**
 * struct nfp_tun_req_route_ipv4 - NFP requests a route/neighbour lookup
 * @ingress_port:       ingress port of packet that signalled request
 * @ipv4_addr:          destination ipv4 address for route
 * @reserved:           reserved for future use
 */
struct nfp_tun_req_route_ipv4 {
        __be32 ingress_port;
        __be32 ipv4_addr;
        __be32 reserved[2];
};

/**
 * struct nfp_tun_req_route_ipv6 - NFP requests an IPv6 route/neighbour lookup
 * @ingress_port:       ingress port of packet that signalled request
 * @ipv6_addr:          destination ipv6 address for route
 */
struct nfp_tun_req_route_ipv6 {
        __be32 ingress_port;
        struct in6_addr ipv6_addr;
};

/**
 * struct nfp_offloaded_route - routes that are offloaded to the NFP
 * @list:       list pointer
 * @ip_add:     destination of route - can be IPv4 or IPv6
 */
struct nfp_offloaded_route {
        struct list_head list;
        u8 ip_add[];
};

#define NFP_FL_IPV4_ADDRS_MAX        32

/**
 * struct nfp_tun_ipv4_addr - set the IP address list on the NFP
 * @count:      number of IPs populated in the array
 * @ipv4_addr:  array of IPV4_ADDRS_MAX 32 bit IPv4 addresses
 */
struct nfp_tun_ipv4_addr {
        __be32 count;
        __be32 ipv4_addr[NFP_FL_IPV4_ADDRS_MAX];
};

/**
 * struct nfp_ipv4_addr_entry - cached IPv4 addresses
 * @ipv4_addr:  IP address
 * @ref_count:  number of rules currently using this IP
 * @list:       list pointer
 */
struct nfp_ipv4_addr_entry {
        __be32 ipv4_addr;
        int ref_count;
        struct list_head list;
};

#define NFP_FL_IPV6_ADDRS_MAX        4

/**
 * struct nfp_tun_ipv6_addr - set the IP address list on the NFP
 * @count:      number of IPs populated in the array
 * @ipv6_addr:  array of IPV6_ADDRS_MAX 128 bit IPv6 addresses
 */
struct nfp_tun_ipv6_addr {
        __be32 count;
        struct in6_addr ipv6_addr[NFP_FL_IPV6_ADDRS_MAX];
};

#define NFP_TUN_MAC_OFFLOAD_DEL_FLAG    0x2

/**
 * struct nfp_tun_mac_addr_offload - configure MAC address of tunnel EP on NFP
 * @flags:      MAC address offload options
 * @count:      number of MAC addresses in the message (should be 1)
 * @index:      index of MAC address in the lookup table
 * @addr:       interface MAC address
 */
struct nfp_tun_mac_addr_offload {
        __be16 flags;
        __be16 count;
        __be16 index;
        u8 addr[ETH_ALEN];
};

enum nfp_flower_mac_offload_cmd {
        NFP_TUNNEL_MAC_OFFLOAD_ADD =            0,
        NFP_TUNNEL_MAC_OFFLOAD_DEL =            1,
        NFP_TUNNEL_MAC_OFFLOAD_MOD =            2,
};

#define NFP_MAX_MAC_INDEX       0xff

/**
 * struct nfp_tun_offloaded_mac - hashtable entry for an offloaded MAC
 * @ht_node:            Hashtable entry
 * @addr:               Offloaded MAC address
 * @index:              Offloaded index for given MAC address
 * @ref_count:          Number of devs using this MAC address
 * @repr_list:          List of reprs sharing this MAC address
 * @bridge_count:       Number of bridge/internal devs with MAC
 */
struct nfp_tun_offloaded_mac {
        struct rhash_head ht_node;
        u8 addr[ETH_ALEN];
        u16 index;
        int ref_count;
        struct list_head repr_list;
        int bridge_count;
};

static const struct rhashtable_params offloaded_macs_params = {
        .key_offset     = offsetof(struct nfp_tun_offloaded_mac, addr),
        .head_offset    = offsetof(struct nfp_tun_offloaded_mac, ht_node),
        .key_len        = ETH_ALEN,
        .automatic_shrinking    = true,
};

void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb)
{
        struct nfp_tun_active_tuns *payload;
        struct net_device *netdev;
        int count, i, pay_len;
        struct neighbour *n;
        __be32 ipv4_addr;
        u32 port;

        payload = nfp_flower_cmsg_get_data(skb);
        count = be32_to_cpu(payload->count);
        if (count > NFP_FL_MAX_ROUTES) {
                nfp_flower_cmsg_warn(app, "Tunnel keep-alive request exceeds max routes.\n");
                return;
        }

        pay_len = nfp_flower_cmsg_get_data_len(skb);
        if (pay_len != struct_size(payload, tun_info, count)) {
                nfp_flower_cmsg_warn(app, "Corruption in tunnel keep-alive message.\n");
                return;
        }

        rcu_read_lock();
        for (i = 0; i < count; i++) {
                ipv4_addr = payload->tun_info[i].ipv4;
                port = be32_to_cpu(payload->tun_info[i].egress_port);
                netdev = nfp_app_dev_get(app, port, NULL);
                if (!netdev)
                        continue;

                n = neigh_lookup(&arp_tbl, &ipv4_addr, netdev);
                if (!n)
                        continue;

                /* Update the used timestamp of neighbour */
                neigh_event_send(n, NULL);
                neigh_release(n);
        }
        rcu_read_unlock();
}

void nfp_tunnel_keep_alive_v6(struct nfp_app *app, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
        struct nfp_tun_active_tuns_v6 *payload;
        struct net_device *netdev;
        int count, i, pay_len;
        struct neighbour *n;
        void *ipv6_add;
        u32 port;

        payload = nfp_flower_cmsg_get_data(skb);
        count = be32_to_cpu(payload->count);
        if (count > NFP_FL_IPV6_ADDRS_MAX) {
                nfp_flower_cmsg_warn(app, "IPv6 tunnel keep-alive request exceeds max routes.\n");
                return;
        }

        pay_len = nfp_flower_cmsg_get_data_len(skb);
        if (pay_len != struct_size(payload, tun_info, count)) {
                nfp_flower_cmsg_warn(app, "Corruption in tunnel keep-alive message.\n");
                return;
        }

        rcu_read_lock();
        for (i = 0; i < count; i++) {
                ipv6_add = &payload->tun_info[i].ipv6;
                port = be32_to_cpu(payload->tun_info[i].egress_port);
                netdev = nfp_app_dev_get(app, port, NULL);
                if (!netdev)
                        continue;

                n = neigh_lookup(&nd_tbl, ipv6_add, netdev);
                if (!n)
                        continue;

                /* Update the used timestamp of neighbour */
                neigh_event_send(n, NULL);
                neigh_release(n);
        }
        rcu_read_unlock();
#endif
}

static int
nfp_flower_xmit_tun_conf(struct nfp_app *app, u8 mtype, u16 plen, void *pdata,
                         gfp_t flag)
{
        struct sk_buff *skb;
        unsigned char *msg;

        skb = nfp_flower_cmsg_alloc(app, plen, mtype, flag);
        if (!skb)
                return -ENOMEM;

        msg = nfp_flower_cmsg_get_data(skb);
        memcpy(msg, pdata, nfp_flower_cmsg_get_data_len(skb));

        nfp_ctrl_tx(app->ctrl, skb);
        return 0;
}

static bool
__nfp_tun_has_route(struct list_head *route_list, spinlock_t *list_lock,
                    void *add, int add_len)
{
        struct nfp_offloaded_route *entry;

        spin_lock_bh(list_lock);
        list_for_each_entry(entry, route_list, list)
                if (!memcmp(entry->ip_add, add, add_len)) {
                        spin_unlock_bh(list_lock);
                        return true;
                }
        spin_unlock_bh(list_lock);
        return false;
}

static int
__nfp_tun_add_route_to_cache(struct list_head *route_list,
                             spinlock_t *list_lock, void *add, int add_len)
{
        struct nfp_offloaded_route *entry;

        spin_lock_bh(list_lock);
        list_for_each_entry(entry, route_list, list)
                if (!memcmp(entry->ip_add, add, add_len)) {
                        spin_unlock_bh(list_lock);
                        return 0;
                }

        entry = kmalloc(sizeof(*entry) + add_len, GFP_ATOMIC);
        if (!entry) {
                spin_unlock_bh(list_lock);
                return -ENOMEM;
        }

        memcpy(entry->ip_add, add, add_len);
        list_add_tail(&entry->list, route_list);
        spin_unlock_bh(list_lock);

        return 0;
}

static void
__nfp_tun_del_route_from_cache(struct list_head *route_list,
                               spinlock_t *list_lock, void *add, int add_len)
{
        struct nfp_offloaded_route *entry;

        spin_lock_bh(list_lock);
        list_for_each_entry(entry, route_list, list)
                if (!memcmp(entry->ip_add, add, add_len)) {
                        list_del(&entry->list);
                        kfree(entry);
                        break;
                }
        spin_unlock_bh(list_lock);
}

static bool nfp_tun_has_route_v4(struct nfp_app *app, __be32 *ipv4_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        return __nfp_tun_has_route(&priv->tun.neigh_off_list_v4,
                                   &priv->tun.neigh_off_lock_v4, ipv4_addr,
                                   sizeof(*ipv4_addr));
}

static bool
nfp_tun_has_route_v6(struct nfp_app *app, struct in6_addr *ipv6_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        return __nfp_tun_has_route(&priv->tun.neigh_off_list_v6,
                                   &priv->tun.neigh_off_lock_v6, ipv6_addr,
                                   sizeof(*ipv6_addr));
}

static void
nfp_tun_add_route_to_cache_v4(struct nfp_app *app, __be32 *ipv4_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        __nfp_tun_add_route_to_cache(&priv->tun.neigh_off_list_v4,
                                     &priv->tun.neigh_off_lock_v4, ipv4_addr,
                                     sizeof(*ipv4_addr));
}

static void
nfp_tun_add_route_to_cache_v6(struct nfp_app *app, struct in6_addr *ipv6_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        __nfp_tun_add_route_to_cache(&priv->tun.neigh_off_list_v6,
                                     &priv->tun.neigh_off_lock_v6, ipv6_addr,
                                     sizeof(*ipv6_addr));
}

static void
nfp_tun_del_route_from_cache_v4(struct nfp_app *app, __be32 *ipv4_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        __nfp_tun_del_route_from_cache(&priv->tun.neigh_off_list_v4,
                                       &priv->tun.neigh_off_lock_v4, ipv4_addr,
                                       sizeof(*ipv4_addr));
}

static void
nfp_tun_del_route_from_cache_v6(struct nfp_app *app, struct in6_addr *ipv6_addr)
{
        struct nfp_flower_priv *priv = app->priv;

        __nfp_tun_del_route_from_cache(&priv->tun.neigh_off_list_v6,
                                       &priv->tun.neigh_off_lock_v6, ipv6_addr,
                                       sizeof(*ipv6_addr));
}

static void
nfp_tun_write_neigh_v4(struct net_device *netdev, struct nfp_app *app,
                       struct flowi4 *flow, struct neighbour *neigh, gfp_t flag)
{
        struct nfp_tun_neigh payload;
        u32 port_id;

        port_id = nfp_flower_get_port_id_from_netdev(app, netdev);
        if (!port_id)
                return;

        memset(&payload, 0, sizeof(struct nfp_tun_neigh));
        payload.dst_ipv4 = flow->daddr;

        /* If entry has expired send dst IP with all other fields 0. */
        if (!(neigh->nud_state & NUD_VALID) || neigh->dead) {
                nfp_tun_del_route_from_cache_v4(app, &payload.dst_ipv4);
                /* Trigger ARP to verify invalid neighbour state. */
                neigh_event_send(neigh, NULL);
                goto send_msg;
        }

        /* Have a valid neighbour so populate rest of entry. */
        payload.src_ipv4 = flow->saddr;
        ether_addr_copy(payload.src_addr, netdev->dev_addr);
        neigh_ha_snapshot(payload.dst_addr, neigh, netdev);
        payload.port_id = cpu_to_be32(port_id);
        /* Add destination of new route to NFP cache. */
        nfp_tun_add_route_to_cache_v4(app, &payload.dst_ipv4);

send_msg:
        nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_NEIGH,
                                 sizeof(struct nfp_tun_neigh),
                                 (unsigned char *)&payload, flag);
}

static void
nfp_tun_write_neigh_v6(struct net_device *netdev, struct nfp_app *app,
                       struct flowi6 *flow, struct neighbour *neigh, gfp_t flag)
{
        struct nfp_tun_neigh_v6 payload;
        u32 port_id;

        port_id = nfp_flower_get_port_id_from_netdev(app, netdev);
        if (!port_id)
                return;

        memset(&payload, 0, sizeof(struct nfp_tun_neigh_v6));
        payload.dst_ipv6 = flow->daddr;

        /* If entry has expired send dst IP with all other fields 0. */
        if (!(neigh->nud_state & NUD_VALID) || neigh->dead) {
                nfp_tun_del_route_from_cache_v6(app, &payload.dst_ipv6);
                /* Trigger probe to verify invalid neighbour state. */
                neigh_event_send(neigh, NULL);
                goto send_msg;
        }

        /* Have a valid neighbour so populate rest of entry. */
        payload.src_ipv6 = flow->saddr;
        ether_addr_copy(payload.src_addr, netdev->dev_addr);
        neigh_ha_snapshot(payload.dst_addr, neigh, netdev);
        payload.port_id = cpu_to_be32(port_id);
        /* Add destination of new route to NFP cache. */
        nfp_tun_add_route_to_cache_v6(app, &payload.dst_ipv6);

send_msg:
        nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6,
                                 sizeof(struct nfp_tun_neigh_v6),
                                 (unsigned char *)&payload, flag);
}

static int
nfp_tun_neigh_event_handler(struct notifier_block *nb, unsigned long event,
                            void *ptr)
{
        struct nfp_flower_priv *app_priv;
        struct netevent_redirect *redir;
        struct flowi4 flow4 = {};
        struct flowi6 flow6 = {};
        struct neighbour *n;
        struct nfp_app *app;
        struct rtable *rt;
        bool ipv6 = false;
        int err;

        switch (event) {
        case NETEVENT_REDIRECT:
                redir = (struct netevent_redirect *)ptr;
                n = redir->neigh;
                break;
        case NETEVENT_NEIGH_UPDATE:
                n = (struct neighbour *)ptr;
                break;
        default:
                return NOTIFY_DONE;
        }

        if (n->tbl->family == AF_INET6)
                ipv6 = true;

        if (ipv6)
                flow6.daddr = *(struct in6_addr *)n->primary_key;
        else
                flow4.daddr = *(__be32 *)n->primary_key;

        app_priv = container_of(nb, struct nfp_flower_priv, tun.neigh_nb);
        app = app_priv->app;

        if (!nfp_netdev_is_nfp_repr(n->dev) &&
            !nfp_flower_internal_port_can_offload(app, n->dev))
                return NOTIFY_DONE;

        /* Only concerned with changes to routes already added to NFP. */
        if ((ipv6 && !nfp_tun_has_route_v6(app, &flow6.daddr)) ||
            (!ipv6 && !nfp_tun_has_route_v4(app, &flow4.daddr)))
                return NOTIFY_DONE;

#if IS_ENABLED(CONFIG_INET)
        if (ipv6) {
#if IS_ENABLED(CONFIG_IPV6)
                struct dst_entry *dst;

                dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(n->dev), NULL,
                                                      &flow6, NULL);
                if (IS_ERR(dst))
                        return NOTIFY_DONE;

                dst_release(dst);
                flow6.flowi6_proto = IPPROTO_UDP;
                nfp_tun_write_neigh_v6(n->dev, app, &flow6, n, GFP_ATOMIC);
#else
                return NOTIFY_DONE;
#endif /* CONFIG_IPV6 */
        } else {
                /* Do a route lookup to populate flow data. */
                rt = ip_route_output_key(dev_net(n->dev), &flow4);
                err = PTR_ERR_OR_ZERO(rt);
                if (err)
                        return NOTIFY_DONE;

                ip_rt_put(rt);

                flow4.flowi4_proto = IPPROTO_UDP;
                nfp_tun_write_neigh_v4(n->dev, app, &flow4, n, GFP_ATOMIC);
        }
#else
        return NOTIFY_DONE;
#endif /* CONFIG_INET */

        return NOTIFY_OK;
}

void nfp_tunnel_request_route_v4(struct nfp_app *app, struct sk_buff *skb)
{
        struct nfp_tun_req_route_ipv4 *payload;
        struct net_device *netdev;
        struct flowi4 flow = {};
        struct neighbour *n;
        struct rtable *rt;
        int err;

        payload = nfp_flower_cmsg_get_data(skb);

        rcu_read_lock();
        netdev = nfp_app_dev_get(app, be32_to_cpu(payload->ingress_port), NULL);
        if (!netdev)
                goto fail_rcu_unlock;

        flow.daddr = payload->ipv4_addr;
        flow.flowi4_proto = IPPROTO_UDP;

#if IS_ENABLED(CONFIG_INET)
        /* Do a route lookup on same namespace as ingress port. */
        rt = ip_route_output_key(dev_net(netdev), &flow);
        err = PTR_ERR_OR_ZERO(rt);
        if (err)
                goto fail_rcu_unlock;
#else
        goto fail_rcu_unlock;
#endif

        /* Get the neighbour entry for the lookup */
        n = dst_neigh_lookup(&rt->dst, &flow.daddr);
        ip_rt_put(rt);
        if (!n)
                goto fail_rcu_unlock;
        nfp_tun_write_neigh_v4(n->dev, app, &flow, n, GFP_ATOMIC);
        neigh_release(n);
        rcu_read_unlock();
        return;

fail_rcu_unlock:
        rcu_read_unlock();
        nfp_flower_cmsg_warn(app, "Requested route not found.\n");
}

void nfp_tunnel_request_route_v6(struct nfp_app *app, struct sk_buff *skb)
{
        struct nfp_tun_req_route_ipv6 *payload;
        struct net_device *netdev;
        struct flowi6 flow = {};
        struct dst_entry *dst;
        struct neighbour *n;

        payload = nfp_flower_cmsg_get_data(skb);

        rcu_read_lock();
        netdev = nfp_app_dev_get(app, be32_to_cpu(payload->ingress_port), NULL);
        if (!netdev)
                goto fail_rcu_unlock;

        flow.daddr = payload->ipv6_addr;
        flow.flowi6_proto = IPPROTO_UDP;

#if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
        dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(netdev), NULL, &flow,
                                              NULL);
        if (IS_ERR(dst))
                goto fail_rcu_unlock;
#else
        goto fail_rcu_unlock;
#endif

        n = dst_neigh_lookup(dst, &flow.daddr);
        dst_release(dst);
        if (!n)
                goto fail_rcu_unlock;

        nfp_tun_write_neigh_v6(n->dev, app, &flow, n, GFP_ATOMIC);
        neigh_release(n);
        rcu_read_unlock();
        return;

fail_rcu_unlock:
        rcu_read_unlock();
        nfp_flower_cmsg_warn(app, "Requested IPv6 route not found.\n");
}

static void nfp_tun_write_ipv4_list(struct nfp_app *app)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv4_addr_entry *entry;
        struct nfp_tun_ipv4_addr payload;
        struct list_head *ptr, *storage;
        int count;

        memset(&payload, 0, sizeof(struct nfp_tun_ipv4_addr));
        mutex_lock(&priv->tun.ipv4_off_lock);
        count = 0;
        list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
                if (count >= NFP_FL_IPV4_ADDRS_MAX) {
                        mutex_unlock(&priv->tun.ipv4_off_lock);
                        nfp_flower_cmsg_warn(app, "IPv4 offload exceeds limit.\n");
                        return;
                }
                entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
                payload.ipv4_addr[count++] = entry->ipv4_addr;
        }
        payload.count = cpu_to_be32(count);
        mutex_unlock(&priv->tun.ipv4_off_lock);

        nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_IPS,
                                 sizeof(struct nfp_tun_ipv4_addr),
                                 &payload, GFP_KERNEL);
}

void nfp_tunnel_add_ipv4_off(struct nfp_app *app, __be32 ipv4)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv4_addr_entry *entry;
        struct list_head *ptr, *storage;

        mutex_lock(&priv->tun.ipv4_off_lock);
        list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
                entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
                if (entry->ipv4_addr == ipv4) {
                        entry->ref_count++;
                        mutex_unlock(&priv->tun.ipv4_off_lock);
                        return;
                }
        }

        entry = kmalloc(sizeof(*entry), GFP_KERNEL);
        if (!entry) {
                mutex_unlock(&priv->tun.ipv4_off_lock);
                nfp_flower_cmsg_warn(app, "Mem error when offloading IP address.\n");
                return;
        }
        entry->ipv4_addr = ipv4;
        entry->ref_count = 1;
        list_add_tail(&entry->list, &priv->tun.ipv4_off_list);
        mutex_unlock(&priv->tun.ipv4_off_lock);

        nfp_tun_write_ipv4_list(app);
}

void nfp_tunnel_del_ipv4_off(struct nfp_app *app, __be32 ipv4)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv4_addr_entry *entry;
        struct list_head *ptr, *storage;

        mutex_lock(&priv->tun.ipv4_off_lock);
        list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
                entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
                if (entry->ipv4_addr == ipv4) {
                        entry->ref_count--;
                        if (!entry->ref_count) {
                                list_del(&entry->list);
                                kfree(entry);
                        }
                        break;
                }
        }
        mutex_unlock(&priv->tun.ipv4_off_lock);

        nfp_tun_write_ipv4_list(app);
}

static void nfp_tun_write_ipv6_list(struct nfp_app *app)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv6_addr_entry *entry;
        struct nfp_tun_ipv6_addr payload;
        int count = 0;

        memset(&payload, 0, sizeof(struct nfp_tun_ipv6_addr));
        mutex_lock(&priv->tun.ipv6_off_lock);
        list_for_each_entry(entry, &priv->tun.ipv6_off_list, list) {
                if (count >= NFP_FL_IPV6_ADDRS_MAX) {
                        nfp_flower_cmsg_warn(app, "Too many IPv6 tunnel endpoint addresses, some cannot be offloaded.\n");
                        break;
                }
                payload.ipv6_addr[count++] = entry->ipv6_addr;
        }
        mutex_unlock(&priv->tun.ipv6_off_lock);
        payload.count = cpu_to_be32(count);

        nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_IPS_V6,
                                 sizeof(struct nfp_tun_ipv6_addr),
                                 &payload, GFP_KERNEL);
}

struct nfp_ipv6_addr_entry *
nfp_tunnel_add_ipv6_off(struct nfp_app *app, struct in6_addr *ipv6)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv6_addr_entry *entry;

        mutex_lock(&priv->tun.ipv6_off_lock);
        list_for_each_entry(entry, &priv->tun.ipv6_off_list, list)
                if (!memcmp(&entry->ipv6_addr, ipv6, sizeof(*ipv6))) {
                        entry->ref_count++;
                        mutex_unlock(&priv->tun.ipv6_off_lock);
                        return entry;
                }

        entry = kmalloc(sizeof(*entry), GFP_KERNEL);
        if (!entry) {
                mutex_unlock(&priv->tun.ipv6_off_lock);
                nfp_flower_cmsg_warn(app, "Mem error when offloading IP address.\n");
                return NULL;
        }
        entry->ipv6_addr = *ipv6;
        entry->ref_count = 1;
        list_add_tail(&entry->list, &priv->tun.ipv6_off_list);
        mutex_unlock(&priv->tun.ipv6_off_lock);

        nfp_tun_write_ipv6_list(app);

        return entry;
}

void
nfp_tunnel_put_ipv6_off(struct nfp_app *app, struct nfp_ipv6_addr_entry *entry)
{
        struct nfp_flower_priv *priv = app->priv;
        bool freed = false;

        mutex_lock(&priv->tun.ipv6_off_lock);
        if (!--entry->ref_count) {
                list_del(&entry->list);
                kfree(entry);
                freed = true;
        }
        mutex_unlock(&priv->tun.ipv6_off_lock);

        if (freed)
                nfp_tun_write_ipv6_list(app);
}

static int
__nfp_tunnel_offload_mac(struct nfp_app *app, u8 *mac, u16 idx, bool del)
{
        struct nfp_tun_mac_addr_offload payload;

        memset(&payload, 0, sizeof(payload));

        if (del)
                payload.flags = cpu_to_be16(NFP_TUN_MAC_OFFLOAD_DEL_FLAG);

        /* FW supports multiple MACs per cmsg but restrict to single. */
        payload.count = cpu_to_be16(1);
        payload.index = cpu_to_be16(idx);
        ether_addr_copy(payload.addr, mac);

        return nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_MAC,
                                        sizeof(struct nfp_tun_mac_addr_offload),
                                        &payload, GFP_KERNEL);
}

static bool nfp_tunnel_port_is_phy_repr(int port)
{
        if (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port) ==
            NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT)
                return true;

        return false;
}

static u16 nfp_tunnel_get_mac_idx_from_phy_port_id(int port)
{
        return port << 8 | NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT;
}

static u16 nfp_tunnel_get_global_mac_idx_from_ida(int id)
{
        return id << 8 | NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT;
}

static int nfp_tunnel_get_ida_from_global_mac_idx(u16 nfp_mac_idx)
{
        return nfp_mac_idx >> 8;
}

static bool nfp_tunnel_is_mac_idx_global(u16 nfp_mac_idx)
{
        return (nfp_mac_idx & 0xff) == NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT;
}

static struct nfp_tun_offloaded_mac *
nfp_tunnel_lookup_offloaded_macs(struct nfp_app *app, u8 *mac)
{
        struct nfp_flower_priv *priv = app->priv;

        return rhashtable_lookup_fast(&priv->tun.offloaded_macs, mac,
                                      offloaded_macs_params);
}

static void
nfp_tunnel_offloaded_macs_inc_ref_and_link(struct nfp_tun_offloaded_mac *entry,
                                           struct net_device *netdev, bool mod)
{
        if (nfp_netdev_is_nfp_repr(netdev)) {
                struct nfp_flower_repr_priv *repr_priv;
                struct nfp_repr *repr;

                repr = netdev_priv(netdev);
                repr_priv = repr->app_priv;

                /* If modifing MAC, remove repr from old list first. */
                if (mod)
                        list_del(&repr_priv->mac_list);

                list_add_tail(&repr_priv->mac_list, &entry->repr_list);
        } else if (nfp_flower_is_supported_bridge(netdev)) {
                entry->bridge_count++;
        }

        entry->ref_count++;
}

static int
nfp_tunnel_add_shared_mac(struct nfp_app *app, struct net_device *netdev,
                          int port, bool mod)
{
        struct nfp_flower_priv *priv = app->priv;
        int ida_idx = NFP_MAX_MAC_INDEX, err;
        struct nfp_tun_offloaded_mac *entry;
        u16 nfp_mac_idx = 0;

        entry = nfp_tunnel_lookup_offloaded_macs(app, netdev->dev_addr);
        if (entry && nfp_tunnel_is_mac_idx_global(entry->index)) {
                if (entry->bridge_count ||
                    !nfp_flower_is_supported_bridge(netdev)) {
                        nfp_tunnel_offloaded_macs_inc_ref_and_link(entry,
                                                                   netdev, mod);
                        return 0;
                }

                /* MAC is global but matches need to go to pre_tun table. */
                nfp_mac_idx = entry->index | NFP_TUN_PRE_TUN_IDX_BIT;
        }

        if (!nfp_mac_idx) {
                /* Assign a global index if non-repr or MAC is now shared. */
                if (entry || !port) {
                        ida_idx = ida_simple_get(&priv->tun.mac_off_ids, 0,
                                                 NFP_MAX_MAC_INDEX, GFP_KERNEL);
                        if (ida_idx < 0)
                                return ida_idx;

                        nfp_mac_idx =
                                nfp_tunnel_get_global_mac_idx_from_ida(ida_idx);

                        if (nfp_flower_is_supported_bridge(netdev))
                                nfp_mac_idx |= NFP_TUN_PRE_TUN_IDX_BIT;

                } else {
                        nfp_mac_idx =
                                nfp_tunnel_get_mac_idx_from_phy_port_id(port);
                }
        }

        if (!entry) {
                entry = kzalloc(sizeof(*entry), GFP_KERNEL);
                if (!entry) {
                        err = -ENOMEM;
                        goto err_free_ida;
                }

                ether_addr_copy(entry->addr, netdev->dev_addr);
                INIT_LIST_HEAD(&entry->repr_list);

                if (rhashtable_insert_fast(&priv->tun.offloaded_macs,
                                           &entry->ht_node,
                                           offloaded_macs_params)) {
                        err = -ENOMEM;
                        goto err_free_entry;
                }
        }

        err = __nfp_tunnel_offload_mac(app, netdev->dev_addr,
                                       nfp_mac_idx, false);
        if (err) {
                /* If not shared then free. */
                if (!entry->ref_count)
                        goto err_remove_hash;
                goto err_free_ida;
        }

        entry->index = nfp_mac_idx;
        nfp_tunnel_offloaded_macs_inc_ref_and_link(entry, netdev, mod);

        return 0;

err_remove_hash:
        rhashtable_remove_fast(&priv->tun.offloaded_macs, &entry->ht_node,
                               offloaded_macs_params);
err_free_entry:
        kfree(entry);
err_free_ida:
        if (ida_idx != NFP_MAX_MAC_INDEX)
                ida_simple_remove(&priv->tun.mac_off_ids, ida_idx);

        return err;
}

static int
nfp_tunnel_del_shared_mac(struct nfp_app *app, struct net_device *netdev,
                          u8 *mac, bool mod)
{
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_flower_repr_priv *repr_priv;
        struct nfp_tun_offloaded_mac *entry;
        struct nfp_repr *repr;
        int ida_idx;

        entry = nfp_tunnel_lookup_offloaded_macs(app, mac);
        if (!entry)
                return 0;

        entry->ref_count--;
        /* If del is part of a mod then mac_list is still in use elsewheree. */
        if (nfp_netdev_is_nfp_repr(netdev) && !mod) {
                repr = netdev_priv(netdev);
                repr_priv = repr->app_priv;
                list_del(&repr_priv->mac_list);
        }

        if (nfp_flower_is_supported_bridge(netdev)) {
                entry->bridge_count--;

                if (!entry->bridge_count && entry->ref_count) {
                        u16 nfp_mac_idx;

                        nfp_mac_idx = entry->index & ~NFP_TUN_PRE_TUN_IDX_BIT;
                        if (__nfp_tunnel_offload_mac(app, mac, nfp_mac_idx,
                                                     false)) {
                                nfp_flower_cmsg_warn(app, "MAC offload index revert failed on %s.\n",
                                                     netdev_name(netdev));
                                return 0;
                        }

                        entry->index = nfp_mac_idx;
                        return 0;
                }
        }

        /* If MAC is now used by 1 repr set the offloaded MAC index to port. */
        if (entry->ref_count == 1 && list_is_singular(&entry->repr_list)) {
                u16 nfp_mac_idx;
                int port, err;

                repr_priv = list_first_entry(&entry->repr_list,
                                             struct nfp_flower_repr_priv,
                                             mac_list);
                repr = repr_priv->nfp_repr;
                port = nfp_repr_get_port_id(repr->netdev);
                nfp_mac_idx = nfp_tunnel_get_mac_idx_from_phy_port_id(port);
                err = __nfp_tunnel_offload_mac(app, mac, nfp_mac_idx, false);
                if (err) {
                        nfp_flower_cmsg_warn(app, "MAC offload index revert failed on %s.\n",
                                             netdev_name(netdev));
                        return 0;
                }

                ida_idx = nfp_tunnel_get_ida_from_global_mac_idx(entry->index);
                ida_simple_remove(&priv->tun.mac_off_ids, ida_idx);
                entry->index = nfp_mac_idx;
                return 0;
        }

        if (entry->ref_count)
                return 0;

        WARN_ON_ONCE(rhashtable_remove_fast(&priv->tun.offloaded_macs,
                                            &entry->ht_node,
                                            offloaded_macs_params));
        /* If MAC has global ID then extract and free the ida entry. */
        if (nfp_tunnel_is_mac_idx_global(entry->index)) {
                ida_idx = nfp_tunnel_get_ida_from_global_mac_idx(entry->index);
                ida_simple_remove(&priv->tun.mac_off_ids, ida_idx);
        }

        kfree(entry);

        return __nfp_tunnel_offload_mac(app, mac, 0, true);
}

static int
nfp_tunnel_offload_mac(struct nfp_app *app, struct net_device *netdev,
                       enum nfp_flower_mac_offload_cmd cmd)
{
        struct nfp_flower_non_repr_priv *nr_priv = NULL;
        bool non_repr = false, *mac_offloaded;
        u8 *off_mac = NULL;
        int err, port = 0;

        if (nfp_netdev_is_nfp_repr(netdev)) {
                struct nfp_flower_repr_priv *repr_priv;
                struct nfp_repr *repr;

                repr = netdev_priv(netdev);
                if (repr->app != app)
                        return 0;

                repr_priv = repr->app_priv;
                if (repr_priv->on_bridge)
                        return 0;

                mac_offloaded = &repr_priv->mac_offloaded;
                off_mac = &repr_priv->offloaded_mac_addr[0];
                port = nfp_repr_get_port_id(netdev);
                if (!nfp_tunnel_port_is_phy_repr(port))
                        return 0;
        } else if (nfp_fl_is_netdev_to_offload(netdev)) {
                nr_priv = nfp_flower_non_repr_priv_get(app, netdev);
                if (!nr_priv)
                        return -ENOMEM;

                mac_offloaded = &nr_priv->mac_offloaded;
                off_mac = &nr_priv->offloaded_mac_addr[0];
                non_repr = true;
        } else {
                return 0;
        }

        if (!is_valid_ether_addr(netdev->dev_addr)) {
                err = -EINVAL;
                goto err_put_non_repr_priv;
        }

        if (cmd == NFP_TUNNEL_MAC_OFFLOAD_MOD && !*mac_offloaded)
                cmd = NFP_TUNNEL_MAC_OFFLOAD_ADD;

        switch (cmd) {
        case NFP_TUNNEL_MAC_OFFLOAD_ADD:
                err = nfp_tunnel_add_shared_mac(app, netdev, port, false);
                if (err)
                        goto err_put_non_repr_priv;

                if (non_repr)
                        __nfp_flower_non_repr_priv_get(nr_priv);

                *mac_offloaded = true;
                ether_addr_copy(off_mac, netdev->dev_addr);
                break;
        case NFP_TUNNEL_MAC_OFFLOAD_DEL:
                /* Only attempt delete if add was successful. */
                if (!*mac_offloaded)
                        break;

                if (non_repr)
                        __nfp_flower_non_repr_priv_put(nr_priv);

                *mac_offloaded = false;

                err = nfp_tunnel_del_shared_mac(app, netdev, netdev->dev_addr,
                                                false);
                if (err)
                        goto err_put_non_repr_priv;

                break;
        case NFP_TUNNEL_MAC_OFFLOAD_MOD:
                /* Ignore if changing to the same address. */
                if (ether_addr_equal(netdev->dev_addr, off_mac))
                        break;

                err = nfp_tunnel_add_shared_mac(app, netdev, port, true);
                if (err)
                        goto err_put_non_repr_priv;

                /* Delete the previous MAC address. */
                err = nfp_tunnel_del_shared_mac(app, netdev, off_mac, true);
                if (err)
                        nfp_flower_cmsg_warn(app, "Failed to remove offload of replaced MAC addr on %s.\n",
                                             netdev_name(netdev));

                ether_addr_copy(off_mac, netdev->dev_addr);
                break;
        default:
                err = -EINVAL;
                goto err_put_non_repr_priv;
        }

        if (non_repr)
                __nfp_flower_non_repr_priv_put(nr_priv);

        return 0;

err_put_non_repr_priv:
        if (non_repr)
                __nfp_flower_non_repr_priv_put(nr_priv);

        return err;
}

int nfp_tunnel_mac_event_handler(struct nfp_app *app,
                                 struct net_device *netdev,
                                 unsigned long event, void *ptr)
{
        int err;

        if (event == NETDEV_DOWN) {
                err = nfp_tunnel_offload_mac(app, netdev,
                                             NFP_TUNNEL_MAC_OFFLOAD_DEL);
                if (err)
                        nfp_flower_cmsg_warn(app, "Failed to delete offload MAC on %s.\n",
                                             netdev_name(netdev));
        } else if (event == NETDEV_UP) {
                err = nfp_tunnel_offload_mac(app, netdev,
                                             NFP_TUNNEL_MAC_OFFLOAD_ADD);
                if (err)
                        nfp_flower_cmsg_warn(app, "Failed to offload MAC on %s.\n",
                                             netdev_name(netdev));
        } else if (event == NETDEV_CHANGEADDR) {
                /* Only offload addr change if netdev is already up. */
                if (!(netdev->flags & IFF_UP))
                        return NOTIFY_OK;

                err = nfp_tunnel_offload_mac(app, netdev,
                                             NFP_TUNNEL_MAC_OFFLOAD_MOD);
                if (err)
                        nfp_flower_cmsg_warn(app, "Failed to offload MAC change on %s.\n",
                                             netdev_name(netdev));
        } else if (event == NETDEV_CHANGEUPPER) {
                /* If a repr is attached to a bridge then tunnel packets
                 * entering the physical port are directed through the bridge
                 * datapath and cannot be directly detunneled. Therefore,
                 * associated offloaded MACs and indexes should not be used
                 * by fw for detunneling.
                 */
                struct netdev_notifier_changeupper_info *info = ptr;
                struct net_device *upper = info->upper_dev;
                struct nfp_flower_repr_priv *repr_priv;
                struct nfp_repr *repr;

                if (!nfp_netdev_is_nfp_repr(netdev) ||
                    !nfp_flower_is_supported_bridge(upper))
                        return NOTIFY_OK;

                repr = netdev_priv(netdev);
                if (repr->app != app)
                        return NOTIFY_OK;

                repr_priv = repr->app_priv;

                if (info->linking) {
                        if (nfp_tunnel_offload_mac(app, netdev,
                                                   NFP_TUNNEL_MAC_OFFLOAD_DEL))
                                nfp_flower_cmsg_warn(app, "Failed to delete offloaded MAC on %s.\n",
                                                     netdev_name(netdev));
                        repr_priv->on_bridge = true;
                } else {
                        repr_priv->on_bridge = false;

                        if (!(netdev->flags & IFF_UP))
                                return NOTIFY_OK;

                        if (nfp_tunnel_offload_mac(app, netdev,
                                                   NFP_TUNNEL_MAC_OFFLOAD_ADD))
                                nfp_flower_cmsg_warn(app, "Failed to offload MAC on %s.\n",
                                                     netdev_name(netdev));
                }
        }
        return NOTIFY_OK;
}

int nfp_flower_xmit_pre_tun_flow(struct nfp_app *app,
                                 struct nfp_fl_payload *flow)
{
        struct nfp_flower_priv *app_priv = app->priv;
        struct nfp_tun_offloaded_mac *mac_entry;
        struct nfp_tun_pre_tun_rule payload;
        struct net_device *internal_dev;
        int err;

        if (app_priv->pre_tun_rule_cnt == NFP_TUN_PRE_TUN_RULE_LIMIT)
                return -ENOSPC;

        memset(&payload, 0, sizeof(struct nfp_tun_pre_tun_rule));

        internal_dev = flow->pre_tun_rule.dev;
        payload.vlan_tci = flow->pre_tun_rule.vlan_tci;
        payload.host_ctx_id = flow->meta.host_ctx_id;

        /* Lookup MAC index for the pre-tunnel rule egress device.
         * Note that because the device is always an internal port, it will
         * have a constant global index so does not need to be tracked.
         */
        mac_entry = nfp_tunnel_lookup_offloaded_macs(app,
                                                     internal_dev->dev_addr);
        if (!mac_entry)
                return -ENOENT;

        payload.port_idx = cpu_to_be16(mac_entry->index);

        /* Copy mac id and vlan to flow - dev may not exist at delete time. */
        flow->pre_tun_rule.vlan_tci = payload.vlan_tci;
        flow->pre_tun_rule.port_idx = payload.port_idx;

        err = nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_PRE_TUN_RULE,
                                       sizeof(struct nfp_tun_pre_tun_rule),
                                       (unsigned char *)&payload, GFP_KERNEL);
        if (err)
                return err;

        app_priv->pre_tun_rule_cnt++;

        return 0;
}

int nfp_flower_xmit_pre_tun_del_flow(struct nfp_app *app,
                                     struct nfp_fl_payload *flow)
{
        struct nfp_flower_priv *app_priv = app->priv;
        struct nfp_tun_pre_tun_rule payload;
        u32 tmp_flags = 0;
        int err;

        memset(&payload, 0, sizeof(struct nfp_tun_pre_tun_rule));

        tmp_flags |= NFP_TUN_PRE_TUN_RULE_DEL;
        payload.flags = cpu_to_be32(tmp_flags);
        payload.vlan_tci = flow->pre_tun_rule.vlan_tci;
        payload.port_idx = flow->pre_tun_rule.port_idx;

        err = nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_PRE_TUN_RULE,
                                       sizeof(struct nfp_tun_pre_tun_rule),
                                       (unsigned char *)&payload, GFP_KERNEL);
        if (err)
                return err;

        app_priv->pre_tun_rule_cnt--;

        return 0;
}

int nfp_tunnel_config_start(struct nfp_app *app)
{
        struct nfp_flower_priv *priv = app->priv;
        int err;

        /* Initialise rhash for MAC offload tracking. */
        err = rhashtable_init(&priv->tun.offloaded_macs,
                              &offloaded_macs_params);
        if (err)
                return err;

        ida_init(&priv->tun.mac_off_ids);

        /* Initialise priv data for IPv4/v6 offloading. */
        mutex_init(&priv->tun.ipv4_off_lock);
        INIT_LIST_HEAD(&priv->tun.ipv4_off_list);
        mutex_init(&priv->tun.ipv6_off_lock);
        INIT_LIST_HEAD(&priv->tun.ipv6_off_list);

        /* Initialise priv data for neighbour offloading. */
        spin_lock_init(&priv->tun.neigh_off_lock_v4);
        INIT_LIST_HEAD(&priv->tun.neigh_off_list_v4);
        spin_lock_init(&priv->tun.neigh_off_lock_v6);
        INIT_LIST_HEAD(&priv->tun.neigh_off_list_v6);
        priv->tun.neigh_nb.notifier_call = nfp_tun_neigh_event_handler;

        err = register_netevent_notifier(&priv->tun.neigh_nb);
        if (err) {
                rhashtable_free_and_destroy(&priv->tun.offloaded_macs,
                                            nfp_check_rhashtable_empty, NULL);
                return err;
        }

        return 0;
}

void nfp_tunnel_config_stop(struct nfp_app *app)
{
        struct nfp_offloaded_route *route_entry, *temp;
        struct nfp_flower_priv *priv = app->priv;
        struct nfp_ipv4_addr_entry *ip_entry;
        struct nfp_tun_neigh_v6 ipv6_route;
        struct nfp_tun_neigh ipv4_route;
        struct list_head *ptr, *storage;

        unregister_netevent_notifier(&priv->tun.neigh_nb);

        ida_destroy(&priv->tun.mac_off_ids);

        /* Free any memory that may be occupied by ipv4 list. */
        list_for_each_safe(ptr, storage, &priv->tun.ipv4_off_list) {
                ip_entry = list_entry(ptr, struct nfp_ipv4_addr_entry, list);
                list_del(&ip_entry->list);
                kfree(ip_entry);
        }

        mutex_destroy(&priv->tun.ipv6_off_lock);

        /* Free memory in the route list and remove entries from fw cache. */
        list_for_each_entry_safe(route_entry, temp,
                                 &priv->tun.neigh_off_list_v4, list) {
                memset(&ipv4_route, 0, sizeof(ipv4_route));
                memcpy(&ipv4_route.dst_ipv4, &route_entry->ip_add,
                       sizeof(ipv4_route.dst_ipv4));
                list_del(&route_entry->list);
                kfree(route_entry);

                nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_NEIGH,
                                         sizeof(struct nfp_tun_neigh),
                                         (unsigned char *)&ipv4_route,
                                         GFP_KERNEL);
        }

        list_for_each_entry_safe(route_entry, temp,
                                 &priv->tun.neigh_off_list_v6, list) {
                memset(&ipv6_route, 0, sizeof(ipv6_route));
                memcpy(&ipv6_route.dst_ipv6, &route_entry->ip_add,
                       sizeof(ipv6_route.dst_ipv6));
                list_del(&route_entry->list);
                kfree(route_entry);

                nfp_flower_xmit_tun_conf(app, NFP_FLOWER_CMSG_TYPE_TUN_NEIGH_V6,
                                         sizeof(struct nfp_tun_neigh),
                                         (unsigned char *)&ipv6_route,
                                         GFP_KERNEL);
        }

        /* Destroy rhash. Entries should be cleaned on netdev notifier unreg. */
        rhashtable_free_and_destroy(&priv->tun.offloaded_macs,
                                    nfp_check_rhashtable_empty, NULL);
}