linux/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
<<
>>
Prefs
   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4#include <linux/kernel.h>
   5#include <linux/types.h>
   6#include <linux/rhashtable.h>
   7#include <linux/bitops.h>
   8#include <linux/in6.h>
   9#include <linux/notifier.h>
  10#include <linux/inetdevice.h>
  11#include <linux/netdevice.h>
  12#include <linux/if_bridge.h>
  13#include <linux/socket.h>
  14#include <linux/route.h>
  15#include <linux/gcd.h>
  16#include <linux/if_macvlan.h>
  17#include <linux/refcount.h>
  18#include <linux/jhash.h>
  19#include <linux/net_namespace.h>
  20#include <linux/mutex.h>
  21#include <net/netevent.h>
  22#include <net/neighbour.h>
  23#include <net/arp.h>
  24#include <net/ip_fib.h>
  25#include <net/ip6_fib.h>
  26#include <net/nexthop.h>
  27#include <net/fib_rules.h>
  28#include <net/ip_tunnels.h>
  29#include <net/l3mdev.h>
  30#include <net/addrconf.h>
  31#include <net/ndisc.h>
  32#include <net/ipv6.h>
  33#include <net/fib_notifier.h>
  34#include <net/switchdev.h>
  35
  36#include "spectrum.h"
  37#include "core.h"
  38#include "reg.h"
  39#include "spectrum_cnt.h"
  40#include "spectrum_dpipe.h"
  41#include "spectrum_ipip.h"
  42#include "spectrum_mr.h"
  43#include "spectrum_mr_tcam.h"
  44#include "spectrum_router.h"
  45#include "spectrum_span.h"
  46
  47struct mlxsw_sp_fib;
  48struct mlxsw_sp_vr;
  49struct mlxsw_sp_lpm_tree;
  50struct mlxsw_sp_rif_ops;
  51
  52struct mlxsw_sp_rif {
  53        struct list_head nexthop_list;
  54        struct list_head neigh_list;
  55        struct net_device *dev; /* NULL for underlay RIF */
  56        struct mlxsw_sp_fid *fid;
  57        unsigned char addr[ETH_ALEN];
  58        int mtu;
  59        u16 rif_index;
  60        u16 vr_id;
  61        const struct mlxsw_sp_rif_ops *ops;
  62        struct mlxsw_sp *mlxsw_sp;
  63
  64        unsigned int counter_ingress;
  65        bool counter_ingress_valid;
  66        unsigned int counter_egress;
  67        bool counter_egress_valid;
  68};
  69
  70struct mlxsw_sp_rif_params {
  71        struct net_device *dev;
  72        union {
  73                u16 system_port;
  74                u16 lag_id;
  75        };
  76        u16 vid;
  77        bool lag;
  78};
  79
  80struct mlxsw_sp_rif_subport {
  81        struct mlxsw_sp_rif common;
  82        refcount_t ref_count;
  83        union {
  84                u16 system_port;
  85                u16 lag_id;
  86        };
  87        u16 vid;
  88        bool lag;
  89};
  90
  91struct mlxsw_sp_rif_ipip_lb {
  92        struct mlxsw_sp_rif common;
  93        struct mlxsw_sp_rif_ipip_lb_config lb_config;
  94        u16 ul_vr_id; /* Reserved for Spectrum-2. */
  95        u16 ul_rif_id; /* Reserved for Spectrum. */
  96};
  97
  98struct mlxsw_sp_rif_params_ipip_lb {
  99        struct mlxsw_sp_rif_params common;
 100        struct mlxsw_sp_rif_ipip_lb_config lb_config;
 101};
 102
 103struct mlxsw_sp_rif_ops {
 104        enum mlxsw_sp_rif_type type;
 105        size_t rif_size;
 106
 107        void (*setup)(struct mlxsw_sp_rif *rif,
 108                      const struct mlxsw_sp_rif_params *params);
 109        int (*configure)(struct mlxsw_sp_rif *rif);
 110        void (*deconfigure)(struct mlxsw_sp_rif *rif);
 111        struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 112                                         struct netlink_ext_ack *extack);
 113        void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 114};
 115
 116struct mlxsw_sp_router_ops {
 117        int (*init)(struct mlxsw_sp *mlxsw_sp);
 118};
 119
 120static struct mlxsw_sp_rif *
 121mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 122                         const struct net_device *dev);
 123static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 124static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 125static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 126                                  struct mlxsw_sp_lpm_tree *lpm_tree);
 127static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 128                                     const struct mlxsw_sp_fib *fib,
 129                                     u8 tree_id);
 130static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 131                                       const struct mlxsw_sp_fib *fib);
 132
 133static unsigned int *
 134mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 135                           enum mlxsw_sp_rif_counter_dir dir)
 136{
 137        switch (dir) {
 138        case MLXSW_SP_RIF_COUNTER_EGRESS:
 139                return &rif->counter_egress;
 140        case MLXSW_SP_RIF_COUNTER_INGRESS:
 141                return &rif->counter_ingress;
 142        }
 143        return NULL;
 144}
 145
 146static bool
 147mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 148                               enum mlxsw_sp_rif_counter_dir dir)
 149{
 150        switch (dir) {
 151        case MLXSW_SP_RIF_COUNTER_EGRESS:
 152                return rif->counter_egress_valid;
 153        case MLXSW_SP_RIF_COUNTER_INGRESS:
 154                return rif->counter_ingress_valid;
 155        }
 156        return false;
 157}
 158
 159static void
 160mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 161                               enum mlxsw_sp_rif_counter_dir dir,
 162                               bool valid)
 163{
 164        switch (dir) {
 165        case MLXSW_SP_RIF_COUNTER_EGRESS:
 166                rif->counter_egress_valid = valid;
 167                break;
 168        case MLXSW_SP_RIF_COUNTER_INGRESS:
 169                rif->counter_ingress_valid = valid;
 170                break;
 171        }
 172}
 173
 174static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 175                                     unsigned int counter_index, bool enable,
 176                                     enum mlxsw_sp_rif_counter_dir dir)
 177{
 178        char ritr_pl[MLXSW_REG_RITR_LEN];
 179        bool is_egress = false;
 180        int err;
 181
 182        if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 183                is_egress = true;
 184        mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 185        err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 186        if (err)
 187                return err;
 188
 189        mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 190                                    is_egress);
 191        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 192}
 193
 194int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 195                                   struct mlxsw_sp_rif *rif,
 196                                   enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 197{
 198        char ricnt_pl[MLXSW_REG_RICNT_LEN];
 199        unsigned int *p_counter_index;
 200        bool valid;
 201        int err;
 202
 203        valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 204        if (!valid)
 205                return -EINVAL;
 206
 207        p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 208        if (!p_counter_index)
 209                return -EINVAL;
 210        mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 211                             MLXSW_REG_RICNT_OPCODE_NOP);
 212        err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 213        if (err)
 214                return err;
 215        *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 216        return 0;
 217}
 218
 219static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 220                                      unsigned int counter_index)
 221{
 222        char ricnt_pl[MLXSW_REG_RICNT_LEN];
 223
 224        mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 225                             MLXSW_REG_RICNT_OPCODE_CLEAR);
 226        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 227}
 228
 229int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 230                               struct mlxsw_sp_rif *rif,
 231                               enum mlxsw_sp_rif_counter_dir dir)
 232{
 233        unsigned int *p_counter_index;
 234        int err;
 235
 236        p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 237        if (!p_counter_index)
 238                return -EINVAL;
 239        err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 240                                     p_counter_index);
 241        if (err)
 242                return err;
 243
 244        err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 245        if (err)
 246                goto err_counter_clear;
 247
 248        err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 249                                        *p_counter_index, true, dir);
 250        if (err)
 251                goto err_counter_edit;
 252        mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 253        return 0;
 254
 255err_counter_edit:
 256err_counter_clear:
 257        mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 258                              *p_counter_index);
 259        return err;
 260}
 261
 262void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 263                               struct mlxsw_sp_rif *rif,
 264                               enum mlxsw_sp_rif_counter_dir dir)
 265{
 266        unsigned int *p_counter_index;
 267
 268        if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 269                return;
 270
 271        p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 272        if (WARN_ON(!p_counter_index))
 273                return;
 274        mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                  *p_counter_index, false, dir);
 276        mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 277                              *p_counter_index);
 278        mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 279}
 280
 281static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 282{
 283        struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 284        struct devlink *devlink;
 285
 286        devlink = priv_to_devlink(mlxsw_sp->core);
 287        if (!devlink_dpipe_table_counter_enabled(devlink,
 288                                                 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 289                return;
 290        mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 291}
 292
 293static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 294{
 295        struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 296
 297        mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 298}
 299
 300#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 301
 302struct mlxsw_sp_prefix_usage {
 303        DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 304};
 305
 306#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 307        for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 308
 309static bool
 310mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 311                         struct mlxsw_sp_prefix_usage *prefix_usage2)
 312{
 313        return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 314}
 315
 316static void
 317mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 318                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 319{
 320        memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 321}
 322
 323static void
 324mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 325                          unsigned char prefix_len)
 326{
 327        set_bit(prefix_len, prefix_usage->b);
 328}
 329
 330static void
 331mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 332                            unsigned char prefix_len)
 333{
 334        clear_bit(prefix_len, prefix_usage->b);
 335}
 336
 337struct mlxsw_sp_fib_key {
 338        unsigned char addr[sizeof(struct in6_addr)];
 339        unsigned char prefix_len;
 340};
 341
 342enum mlxsw_sp_fib_entry_type {
 343        MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 344        MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 345        MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 346        MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 347        MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
 348
 349        /* This is a special case of local delivery, where a packet should be
 350         * decapsulated on reception. Note that there is no corresponding ENCAP,
 351         * because that's a type of next hop, not of FIB entry. (There can be
 352         * several next hops in a REMOTE entry, and some of them may be
 353         * encapsulating entries.)
 354         */
 355        MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 356        MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 357};
 358
 359struct mlxsw_sp_nexthop_group_info;
 360struct mlxsw_sp_nexthop_group;
 361struct mlxsw_sp_fib_entry;
 362
 363struct mlxsw_sp_fib_node {
 364        struct mlxsw_sp_fib_entry *fib_entry;
 365        struct list_head list;
 366        struct rhash_head ht_node;
 367        struct mlxsw_sp_fib *fib;
 368        struct mlxsw_sp_fib_key key;
 369};
 370
 371struct mlxsw_sp_fib_entry_decap {
 372        struct mlxsw_sp_ipip_entry *ipip_entry;
 373        u32 tunnel_index;
 374};
 375
 376static struct mlxsw_sp_fib_entry_priv *
 377mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
 378{
 379        struct mlxsw_sp_fib_entry_priv *priv;
 380
 381        if (!ll_ops->fib_entry_priv_size)
 382                /* No need to have priv */
 383                return NULL;
 384
 385        priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
 386        if (!priv)
 387                return ERR_PTR(-ENOMEM);
 388        refcount_set(&priv->refcnt, 1);
 389        return priv;
 390}
 391
 392static void
 393mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
 394{
 395        kfree(priv);
 396}
 397
 398static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
 399{
 400        refcount_inc(&priv->refcnt);
 401}
 402
 403static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
 404{
 405        if (!priv || !refcount_dec_and_test(&priv->refcnt))
 406                return;
 407        mlxsw_sp_fib_entry_priv_destroy(priv);
 408}
 409
 410static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
 411                                                struct mlxsw_sp_fib_entry_priv *priv)
 412{
 413        if (!priv)
 414                return;
 415        mlxsw_sp_fib_entry_priv_hold(priv);
 416        list_add(&priv->list, &op_ctx->fib_entry_priv_list);
 417}
 418
 419static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
 420{
 421        struct mlxsw_sp_fib_entry_priv *priv, *tmp;
 422
 423        list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
 424                mlxsw_sp_fib_entry_priv_put(priv);
 425        INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
 426}
 427
 428struct mlxsw_sp_fib_entry {
 429        struct mlxsw_sp_fib_node *fib_node;
 430        enum mlxsw_sp_fib_entry_type type;
 431        struct list_head nexthop_group_node;
 432        struct mlxsw_sp_nexthop_group *nh_group;
 433        struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 434        struct mlxsw_sp_fib_entry_priv *priv;
 435};
 436
 437struct mlxsw_sp_fib4_entry {
 438        struct mlxsw_sp_fib_entry common;
 439        struct fib_info *fi;
 440        u32 tb_id;
 441        u8 tos;
 442        u8 type;
 443};
 444
 445struct mlxsw_sp_fib6_entry {
 446        struct mlxsw_sp_fib_entry common;
 447        struct list_head rt6_list;
 448        unsigned int nrt6;
 449};
 450
 451struct mlxsw_sp_rt6 {
 452        struct list_head list;
 453        struct fib6_info *rt;
 454};
 455
 456struct mlxsw_sp_lpm_tree {
 457        u8 id; /* tree ID */
 458        unsigned int ref_count;
 459        enum mlxsw_sp_l3proto proto;
 460        unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 461        struct mlxsw_sp_prefix_usage prefix_usage;
 462};
 463
 464struct mlxsw_sp_fib {
 465        struct rhashtable ht;
 466        struct list_head node_list;
 467        struct mlxsw_sp_vr *vr;
 468        struct mlxsw_sp_lpm_tree *lpm_tree;
 469        enum mlxsw_sp_l3proto proto;
 470        const struct mlxsw_sp_router_ll_ops *ll_ops;
 471};
 472
 473struct mlxsw_sp_vr {
 474        u16 id; /* virtual router ID */
 475        u32 tb_id; /* kernel fib table id */
 476        unsigned int rif_count;
 477        struct mlxsw_sp_fib *fib4;
 478        struct mlxsw_sp_fib *fib6;
 479        struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 480        struct mlxsw_sp_rif *ul_rif;
 481        refcount_t ul_rif_refcnt;
 482};
 483
 484static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
 485                                         enum mlxsw_sp_l3proto proto)
 486{
 487        return 0;
 488}
 489
 490static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
 491{
 492        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
 493                               xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
 494}
 495
 496static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
 497{
 498        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
 499                               xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
 500}
 501
 502static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
 503{
 504        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 505                               xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
 506}
 507
 508static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 509
 510static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 511                                                struct mlxsw_sp_vr *vr,
 512                                                enum mlxsw_sp_l3proto proto)
 513{
 514        const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 515        struct mlxsw_sp_lpm_tree *lpm_tree;
 516        struct mlxsw_sp_fib *fib;
 517        int err;
 518
 519        err = ll_ops->init(mlxsw_sp, vr->id, proto);
 520        if (err)
 521                return ERR_PTR(err);
 522
 523        lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 524        fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 525        if (!fib)
 526                return ERR_PTR(-ENOMEM);
 527        err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 528        if (err)
 529                goto err_rhashtable_init;
 530        INIT_LIST_HEAD(&fib->node_list);
 531        fib->proto = proto;
 532        fib->vr = vr;
 533        fib->lpm_tree = lpm_tree;
 534        fib->ll_ops = ll_ops;
 535        mlxsw_sp_lpm_tree_hold(lpm_tree);
 536        err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 537        if (err)
 538                goto err_lpm_tree_bind;
 539        return fib;
 540
 541err_lpm_tree_bind:
 542        mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 543err_rhashtable_init:
 544        kfree(fib);
 545        return ERR_PTR(err);
 546}
 547
 548static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 549                                 struct mlxsw_sp_fib *fib)
 550{
 551        mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 552        mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 553        WARN_ON(!list_empty(&fib->node_list));
 554        rhashtable_destroy(&fib->ht);
 555        kfree(fib);
 556}
 557
 558static struct mlxsw_sp_lpm_tree *
 559mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 560{
 561        static struct mlxsw_sp_lpm_tree *lpm_tree;
 562        int i;
 563
 564        for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 565                lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 566                if (lpm_tree->ref_count == 0)
 567                        return lpm_tree;
 568        }
 569        return NULL;
 570}
 571
 572static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 573                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 574                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 575{
 576        char xralta_pl[MLXSW_REG_XRALTA_LEN];
 577
 578        mlxsw_reg_xralta_pack(xralta_pl, true,
 579                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 580                              lpm_tree->id);
 581        return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 582}
 583
 584static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 585                                   const struct mlxsw_sp_router_ll_ops *ll_ops,
 586                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 587{
 588        char xralta_pl[MLXSW_REG_XRALTA_LEN];
 589
 590        mlxsw_reg_xralta_pack(xralta_pl, false,
 591                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 592                              lpm_tree->id);
 593        ll_ops->ralta_write(mlxsw_sp, xralta_pl);
 594}
 595
 596static int
 597mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 598                                  const struct mlxsw_sp_router_ll_ops *ll_ops,
 599                                  struct mlxsw_sp_prefix_usage *prefix_usage,
 600                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 601{
 602        char xralst_pl[MLXSW_REG_XRALST_LEN];
 603        u8 root_bin = 0;
 604        u8 prefix;
 605        u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 606
 607        mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 608                root_bin = prefix;
 609
 610        mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
 611        mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 612                if (prefix == 0)
 613                        continue;
 614                mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
 615                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 616                last_prefix = prefix;
 617        }
 618        return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
 619}
 620
 621static struct mlxsw_sp_lpm_tree *
 622mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 623                         const struct mlxsw_sp_router_ll_ops *ll_ops,
 624                         struct mlxsw_sp_prefix_usage *prefix_usage,
 625                         enum mlxsw_sp_l3proto proto)
 626{
 627        struct mlxsw_sp_lpm_tree *lpm_tree;
 628        int err;
 629
 630        lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 631        if (!lpm_tree)
 632                return ERR_PTR(-EBUSY);
 633        lpm_tree->proto = proto;
 634        err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
 635        if (err)
 636                return ERR_PTR(err);
 637
 638        err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
 639        if (err)
 640                goto err_left_struct_set;
 641        memcpy(&lpm_tree->prefix_usage, prefix_usage,
 642               sizeof(lpm_tree->prefix_usage));
 643        memset(&lpm_tree->prefix_ref_count, 0,
 644               sizeof(lpm_tree->prefix_ref_count));
 645        lpm_tree->ref_count = 1;
 646        return lpm_tree;
 647
 648err_left_struct_set:
 649        mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 650        return ERR_PTR(err);
 651}
 652
 653static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 654                                      const struct mlxsw_sp_router_ll_ops *ll_ops,
 655                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 656{
 657        mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
 658}
 659
 660static struct mlxsw_sp_lpm_tree *
 661mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 662                      struct mlxsw_sp_prefix_usage *prefix_usage,
 663                      enum mlxsw_sp_l3proto proto)
 664{
 665        const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
 666        struct mlxsw_sp_lpm_tree *lpm_tree;
 667        int i;
 668
 669        for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 670                lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 671                if (lpm_tree->ref_count != 0 &&
 672                    lpm_tree->proto == proto &&
 673                    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 674                                             prefix_usage)) {
 675                        mlxsw_sp_lpm_tree_hold(lpm_tree);
 676                        return lpm_tree;
 677                }
 678        }
 679        return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
 680}
 681
 682static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 683{
 684        lpm_tree->ref_count++;
 685}
 686
 687static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 688                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 689{
 690        const struct mlxsw_sp_router_ll_ops *ll_ops =
 691                                mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
 692
 693        if (--lpm_tree->ref_count == 0)
 694                mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
 695}
 696
 697#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 698
 699static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 700{
 701        struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 702        struct mlxsw_sp_lpm_tree *lpm_tree;
 703        u64 max_trees;
 704        int err, i;
 705
 706        if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 707                return -EIO;
 708
 709        max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 710        mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 711        mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 712                                             sizeof(struct mlxsw_sp_lpm_tree),
 713                                             GFP_KERNEL);
 714        if (!mlxsw_sp->router->lpm.trees)
 715                return -ENOMEM;
 716
 717        for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 718                lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 719                lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 720        }
 721
 722        lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 723                                         MLXSW_SP_L3_PROTO_IPV4);
 724        if (IS_ERR(lpm_tree)) {
 725                err = PTR_ERR(lpm_tree);
 726                goto err_ipv4_tree_get;
 727        }
 728        mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 729
 730        lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 731                                         MLXSW_SP_L3_PROTO_IPV6);
 732        if (IS_ERR(lpm_tree)) {
 733                err = PTR_ERR(lpm_tree);
 734                goto err_ipv6_tree_get;
 735        }
 736        mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 737
 738        return 0;
 739
 740err_ipv6_tree_get:
 741        lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 742        mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 743err_ipv4_tree_get:
 744        kfree(mlxsw_sp->router->lpm.trees);
 745        return err;
 746}
 747
 748static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 749{
 750        struct mlxsw_sp_lpm_tree *lpm_tree;
 751
 752        lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 753        mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 754
 755        lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 756        mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 757
 758        kfree(mlxsw_sp->router->lpm.trees);
 759}
 760
 761static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 762{
 763        return !!vr->fib4 || !!vr->fib6 ||
 764               !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 765               !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 766}
 767
 768static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 769{
 770        struct mlxsw_sp_vr *vr;
 771        int i;
 772
 773        for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 774                vr = &mlxsw_sp->router->vrs[i];
 775                if (!mlxsw_sp_vr_is_used(vr))
 776                        return vr;
 777        }
 778        return NULL;
 779}
 780
 781static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 782                                     const struct mlxsw_sp_fib *fib, u8 tree_id)
 783{
 784        char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 785
 786        mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 787                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 788                              tree_id);
 789        return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 790}
 791
 792static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 793                                       const struct mlxsw_sp_fib *fib)
 794{
 795        char xraltb_pl[MLXSW_REG_XRALTB_LEN];
 796
 797        /* Bind to tree 0 which is default */
 798        mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
 799                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 800        return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
 801}
 802
 803static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 804{
 805        /* For our purpose, squash main, default and local tables into one */
 806        if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 807                tb_id = RT_TABLE_MAIN;
 808        return tb_id;
 809}
 810
 811static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 812                                            u32 tb_id)
 813{
 814        struct mlxsw_sp_vr *vr;
 815        int i;
 816
 817        tb_id = mlxsw_sp_fix_tb_id(tb_id);
 818
 819        for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 820                vr = &mlxsw_sp->router->vrs[i];
 821                if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 822                        return vr;
 823        }
 824        return NULL;
 825}
 826
 827int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 828                                u16 *vr_id)
 829{
 830        struct mlxsw_sp_vr *vr;
 831        int err = 0;
 832
 833        mutex_lock(&mlxsw_sp->router->lock);
 834        vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 835        if (!vr) {
 836                err = -ESRCH;
 837                goto out;
 838        }
 839        *vr_id = vr->id;
 840out:
 841        mutex_unlock(&mlxsw_sp->router->lock);
 842        return err;
 843}
 844
 845static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 846                                            enum mlxsw_sp_l3proto proto)
 847{
 848        switch (proto) {
 849        case MLXSW_SP_L3_PROTO_IPV4:
 850                return vr->fib4;
 851        case MLXSW_SP_L3_PROTO_IPV6:
 852                return vr->fib6;
 853        }
 854        return NULL;
 855}
 856
 857static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 858                                              u32 tb_id,
 859                                              struct netlink_ext_ack *extack)
 860{
 861        struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 862        struct mlxsw_sp_fib *fib4;
 863        struct mlxsw_sp_fib *fib6;
 864        struct mlxsw_sp_vr *vr;
 865        int err;
 866
 867        vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 868        if (!vr) {
 869                NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 870                return ERR_PTR(-EBUSY);
 871        }
 872        fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 873        if (IS_ERR(fib4))
 874                return ERR_CAST(fib4);
 875        fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 876        if (IS_ERR(fib6)) {
 877                err = PTR_ERR(fib6);
 878                goto err_fib6_create;
 879        }
 880        mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 881                                             MLXSW_SP_L3_PROTO_IPV4);
 882        if (IS_ERR(mr4_table)) {
 883                err = PTR_ERR(mr4_table);
 884                goto err_mr4_table_create;
 885        }
 886        mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 887                                             MLXSW_SP_L3_PROTO_IPV6);
 888        if (IS_ERR(mr6_table)) {
 889                err = PTR_ERR(mr6_table);
 890                goto err_mr6_table_create;
 891        }
 892
 893        vr->fib4 = fib4;
 894        vr->fib6 = fib6;
 895        vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 896        vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 897        vr->tb_id = tb_id;
 898        return vr;
 899
 900err_mr6_table_create:
 901        mlxsw_sp_mr_table_destroy(mr4_table);
 902err_mr4_table_create:
 903        mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 904err_fib6_create:
 905        mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 906        return ERR_PTR(err);
 907}
 908
 909static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 910                                struct mlxsw_sp_vr *vr)
 911{
 912        mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 913        vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 914        mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 915        vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 916        mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 917        vr->fib6 = NULL;
 918        mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 919        vr->fib4 = NULL;
 920}
 921
 922static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 923                                           struct netlink_ext_ack *extack)
 924{
 925        struct mlxsw_sp_vr *vr;
 926
 927        tb_id = mlxsw_sp_fix_tb_id(tb_id);
 928        vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 929        if (!vr)
 930                vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 931        return vr;
 932}
 933
 934static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 935{
 936        if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 937            list_empty(&vr->fib6->node_list) &&
 938            mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 939            mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 940                mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 941}
 942
 943static bool
 944mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 945                                    enum mlxsw_sp_l3proto proto, u8 tree_id)
 946{
 947        struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 948
 949        if (!mlxsw_sp_vr_is_used(vr))
 950                return false;
 951        if (fib->lpm_tree->id == tree_id)
 952                return true;
 953        return false;
 954}
 955
 956static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 957                                        struct mlxsw_sp_fib *fib,
 958                                        struct mlxsw_sp_lpm_tree *new_tree)
 959{
 960        struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 961        int err;
 962
 963        fib->lpm_tree = new_tree;
 964        mlxsw_sp_lpm_tree_hold(new_tree);
 965        err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 966        if (err)
 967                goto err_tree_bind;
 968        mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 969        return 0;
 970
 971err_tree_bind:
 972        mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 973        fib->lpm_tree = old_tree;
 974        return err;
 975}
 976
 977static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 978                                         struct mlxsw_sp_fib *fib,
 979                                         struct mlxsw_sp_lpm_tree *new_tree)
 980{
 981        enum mlxsw_sp_l3proto proto = fib->proto;
 982        struct mlxsw_sp_lpm_tree *old_tree;
 983        u8 old_id, new_id = new_tree->id;
 984        struct mlxsw_sp_vr *vr;
 985        int i, err;
 986
 987        old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 988        old_id = old_tree->id;
 989
 990        for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 991                vr = &mlxsw_sp->router->vrs[i];
 992                if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 993                        continue;
 994                err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 995                                                   mlxsw_sp_vr_fib(vr, proto),
 996                                                   new_tree);
 997                if (err)
 998                        goto err_tree_replace;
 999        }
1000
1001        memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
1002               sizeof(new_tree->prefix_ref_count));
1003        mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
1004        mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
1005
1006        return 0;
1007
1008err_tree_replace:
1009        for (i--; i >= 0; i--) {
1010                if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
1011                        continue;
1012                mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1013                                             mlxsw_sp_vr_fib(vr, proto),
1014                                             old_tree);
1015        }
1016        return err;
1017}
1018
1019static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1020{
1021        struct mlxsw_sp_vr *vr;
1022        u64 max_vrs;
1023        int i;
1024
1025        if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1026                return -EIO;
1027
1028        max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1029        mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1030                                        GFP_KERNEL);
1031        if (!mlxsw_sp->router->vrs)
1032                return -ENOMEM;
1033
1034        for (i = 0; i < max_vrs; i++) {
1035                vr = &mlxsw_sp->router->vrs[i];
1036                vr->id = i;
1037        }
1038
1039        return 0;
1040}
1041
1042static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1043
1044static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1045{
1046        /* At this stage we're guaranteed not to have new incoming
1047         * FIB notifications and the work queue is free from FIBs
1048         * sitting on top of mlxsw netdevs. However, we can still
1049         * have other FIBs queued. Flush the queue before flushing
1050         * the device's tables. No need for locks, as we're the only
1051         * writer.
1052         */
1053        mlxsw_core_flush_owq();
1054        mlxsw_sp_router_fib_flush(mlxsw_sp);
1055        kfree(mlxsw_sp->router->vrs);
1056}
1057
1058static struct net_device *
1059__mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
1060{
1061        struct ip_tunnel *tun = netdev_priv(ol_dev);
1062        struct net *net = dev_net(ol_dev);
1063
1064        return dev_get_by_index_rcu(net, tun->parms.link);
1065}
1066
1067u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1068{
1069        struct net_device *d;
1070        u32 tb_id;
1071
1072        rcu_read_lock();
1073        d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1074        if (d)
1075                tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1076        else
1077                tb_id = RT_TABLE_MAIN;
1078        rcu_read_unlock();
1079
1080        return tb_id;
1081}
1082
1083static struct mlxsw_sp_rif *
1084mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1085                    const struct mlxsw_sp_rif_params *params,
1086                    struct netlink_ext_ack *extack);
1087
1088static struct mlxsw_sp_rif_ipip_lb *
1089mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1090                                enum mlxsw_sp_ipip_type ipipt,
1091                                struct net_device *ol_dev,
1092                                struct netlink_ext_ack *extack)
1093{
1094        struct mlxsw_sp_rif_params_ipip_lb lb_params;
1095        const struct mlxsw_sp_ipip_ops *ipip_ops;
1096        struct mlxsw_sp_rif *rif;
1097
1098        ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1099        lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1100                .common.dev = ol_dev,
1101                .common.lag = false,
1102                .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1103        };
1104
1105        rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1106        if (IS_ERR(rif))
1107                return ERR_CAST(rif);
1108        return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1109}
1110
1111static struct mlxsw_sp_ipip_entry *
1112mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1113                          enum mlxsw_sp_ipip_type ipipt,
1114                          struct net_device *ol_dev)
1115{
1116        const struct mlxsw_sp_ipip_ops *ipip_ops;
1117        struct mlxsw_sp_ipip_entry *ipip_entry;
1118        struct mlxsw_sp_ipip_entry *ret = NULL;
1119
1120        ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1121        ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1122        if (!ipip_entry)
1123                return ERR_PTR(-ENOMEM);
1124
1125        ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1126                                                            ol_dev, NULL);
1127        if (IS_ERR(ipip_entry->ol_lb)) {
1128                ret = ERR_CAST(ipip_entry->ol_lb);
1129                goto err_ol_ipip_lb_create;
1130        }
1131
1132        ipip_entry->ipipt = ipipt;
1133        ipip_entry->ol_dev = ol_dev;
1134
1135        switch (ipip_ops->ul_proto) {
1136        case MLXSW_SP_L3_PROTO_IPV4:
1137                ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1138                break;
1139        case MLXSW_SP_L3_PROTO_IPV6:
1140                WARN_ON(1);
1141                break;
1142        }
1143
1144        return ipip_entry;
1145
1146err_ol_ipip_lb_create:
1147        kfree(ipip_entry);
1148        return ret;
1149}
1150
1151static void
1152mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1153{
1154        mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1155        kfree(ipip_entry);
1156}
1157
1158static bool
1159mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1160                                  const enum mlxsw_sp_l3proto ul_proto,
1161                                  union mlxsw_sp_l3addr saddr,
1162                                  u32 ul_tb_id,
1163                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1164{
1165        u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1166        enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1167        union mlxsw_sp_l3addr tun_saddr;
1168
1169        if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1170                return false;
1171
1172        tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1173        return tun_ul_tb_id == ul_tb_id &&
1174               mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1175}
1176
1177static int
1178mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1179                              struct mlxsw_sp_fib_entry *fib_entry,
1180                              struct mlxsw_sp_ipip_entry *ipip_entry)
1181{
1182        u32 tunnel_index;
1183        int err;
1184
1185        err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1186                                  1, &tunnel_index);
1187        if (err)
1188                return err;
1189
1190        ipip_entry->decap_fib_entry = fib_entry;
1191        fib_entry->decap.ipip_entry = ipip_entry;
1192        fib_entry->decap.tunnel_index = tunnel_index;
1193        return 0;
1194}
1195
1196static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1197                                          struct mlxsw_sp_fib_entry *fib_entry)
1198{
1199        /* Unlink this node from the IPIP entry that it's the decap entry of. */
1200        fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1201        fib_entry->decap.ipip_entry = NULL;
1202        mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1203                           1, fib_entry->decap.tunnel_index);
1204}
1205
1206static struct mlxsw_sp_fib_node *
1207mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1208                         size_t addr_len, unsigned char prefix_len);
1209static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1210                                     struct mlxsw_sp_fib_entry *fib_entry);
1211
1212static void
1213mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1214                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1215{
1216        struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1217
1218        mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1219        fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1220
1221        mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1222}
1223
1224static void
1225mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1226                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1227                                  struct mlxsw_sp_fib_entry *decap_fib_entry)
1228{
1229        if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1230                                          ipip_entry))
1231                return;
1232        decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1233
1234        if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1235                mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1236}
1237
1238static struct mlxsw_sp_fib_entry *
1239mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1240                                     enum mlxsw_sp_l3proto proto,
1241                                     const union mlxsw_sp_l3addr *addr,
1242                                     enum mlxsw_sp_fib_entry_type type)
1243{
1244        struct mlxsw_sp_fib_node *fib_node;
1245        unsigned char addr_prefix_len;
1246        struct mlxsw_sp_fib *fib;
1247        struct mlxsw_sp_vr *vr;
1248        const void *addrp;
1249        size_t addr_len;
1250        u32 addr4;
1251
1252        vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1253        if (!vr)
1254                return NULL;
1255        fib = mlxsw_sp_vr_fib(vr, proto);
1256
1257        switch (proto) {
1258        case MLXSW_SP_L3_PROTO_IPV4:
1259                addr4 = be32_to_cpu(addr->addr4);
1260                addrp = &addr4;
1261                addr_len = 4;
1262                addr_prefix_len = 32;
1263                break;
1264        case MLXSW_SP_L3_PROTO_IPV6:
1265        default:
1266                WARN_ON(1);
1267                return NULL;
1268        }
1269
1270        fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1271                                            addr_prefix_len);
1272        if (!fib_node || fib_node->fib_entry->type != type)
1273                return NULL;
1274
1275        return fib_node->fib_entry;
1276}
1277
1278/* Given an IPIP entry, find the corresponding decap route. */
1279static struct mlxsw_sp_fib_entry *
1280mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1281                               struct mlxsw_sp_ipip_entry *ipip_entry)
1282{
1283        static struct mlxsw_sp_fib_node *fib_node;
1284        const struct mlxsw_sp_ipip_ops *ipip_ops;
1285        unsigned char saddr_prefix_len;
1286        union mlxsw_sp_l3addr saddr;
1287        struct mlxsw_sp_fib *ul_fib;
1288        struct mlxsw_sp_vr *ul_vr;
1289        const void *saddrp;
1290        size_t saddr_len;
1291        u32 ul_tb_id;
1292        u32 saddr4;
1293
1294        ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1295
1296        ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1297        ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1298        if (!ul_vr)
1299                return NULL;
1300
1301        ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1302        saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1303                                           ipip_entry->ol_dev);
1304
1305        switch (ipip_ops->ul_proto) {
1306        case MLXSW_SP_L3_PROTO_IPV4:
1307                saddr4 = be32_to_cpu(saddr.addr4);
1308                saddrp = &saddr4;
1309                saddr_len = 4;
1310                saddr_prefix_len = 32;
1311                break;
1312        default:
1313                WARN_ON(1);
1314                return NULL;
1315        }
1316
1317        fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1318                                            saddr_prefix_len);
1319        if (!fib_node ||
1320            fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1321                return NULL;
1322
1323        return fib_node->fib_entry;
1324}
1325
1326static struct mlxsw_sp_ipip_entry *
1327mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1328                           enum mlxsw_sp_ipip_type ipipt,
1329                           struct net_device *ol_dev)
1330{
1331        struct mlxsw_sp_ipip_entry *ipip_entry;
1332
1333        ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1334        if (IS_ERR(ipip_entry))
1335                return ipip_entry;
1336
1337        list_add_tail(&ipip_entry->ipip_list_node,
1338                      &mlxsw_sp->router->ipip_list);
1339
1340        return ipip_entry;
1341}
1342
1343static void
1344mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1345                            struct mlxsw_sp_ipip_entry *ipip_entry)
1346{
1347        list_del(&ipip_entry->ipip_list_node);
1348        mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1349}
1350
1351static bool
1352mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1353                                  const struct net_device *ul_dev,
1354                                  enum mlxsw_sp_l3proto ul_proto,
1355                                  union mlxsw_sp_l3addr ul_dip,
1356                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1357{
1358        u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1359        enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1360
1361        if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1362                return false;
1363
1364        return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1365                                                 ul_tb_id, ipip_entry);
1366}
1367
1368/* Given decap parameters, find the corresponding IPIP entry. */
1369static struct mlxsw_sp_ipip_entry *
1370mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1371                                  enum mlxsw_sp_l3proto ul_proto,
1372                                  union mlxsw_sp_l3addr ul_dip)
1373{
1374        struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1375        struct net_device *ul_dev;
1376
1377        rcu_read_lock();
1378
1379        ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1380        if (!ul_dev)
1381                goto out_unlock;
1382
1383        list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1384                            ipip_list_node)
1385                if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1386                                                      ul_proto, ul_dip,
1387                                                      ipip_entry))
1388                        goto out_unlock;
1389
1390        rcu_read_unlock();
1391
1392        return NULL;
1393
1394out_unlock:
1395        rcu_read_unlock();
1396        return ipip_entry;
1397}
1398
1399static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1400                                      const struct net_device *dev,
1401                                      enum mlxsw_sp_ipip_type *p_type)
1402{
1403        struct mlxsw_sp_router *router = mlxsw_sp->router;
1404        const struct mlxsw_sp_ipip_ops *ipip_ops;
1405        enum mlxsw_sp_ipip_type ipipt;
1406
1407        for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1408                ipip_ops = router->ipip_ops_arr[ipipt];
1409                if (dev->type == ipip_ops->dev_type) {
1410                        if (p_type)
1411                                *p_type = ipipt;
1412                        return true;
1413                }
1414        }
1415        return false;
1416}
1417
1418bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1419                                const struct net_device *dev)
1420{
1421        return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1422}
1423
1424static struct mlxsw_sp_ipip_entry *
1425mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1426                                   const struct net_device *ol_dev)
1427{
1428        struct mlxsw_sp_ipip_entry *ipip_entry;
1429
1430        list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1431                            ipip_list_node)
1432                if (ipip_entry->ol_dev == ol_dev)
1433                        return ipip_entry;
1434
1435        return NULL;
1436}
1437
1438static struct mlxsw_sp_ipip_entry *
1439mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1440                                   const struct net_device *ul_dev,
1441                                   struct mlxsw_sp_ipip_entry *start)
1442{
1443        struct mlxsw_sp_ipip_entry *ipip_entry;
1444
1445        ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1446                                        ipip_list_node);
1447        list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1448                                     ipip_list_node) {
1449                struct net_device *ol_dev = ipip_entry->ol_dev;
1450                struct net_device *ipip_ul_dev;
1451
1452                rcu_read_lock();
1453                ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1454                rcu_read_unlock();
1455
1456                if (ipip_ul_dev == ul_dev)
1457                        return ipip_entry;
1458        }
1459
1460        return NULL;
1461}
1462
1463bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1464                                const struct net_device *dev)
1465{
1466        bool is_ipip_ul;
1467
1468        mutex_lock(&mlxsw_sp->router->lock);
1469        is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1470        mutex_unlock(&mlxsw_sp->router->lock);
1471
1472        return is_ipip_ul;
1473}
1474
1475static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1476                                                const struct net_device *ol_dev,
1477                                                enum mlxsw_sp_ipip_type ipipt)
1478{
1479        const struct mlxsw_sp_ipip_ops *ops
1480                = mlxsw_sp->router->ipip_ops_arr[ipipt];
1481
1482        return ops->can_offload(mlxsw_sp, ol_dev);
1483}
1484
1485static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1486                                                struct net_device *ol_dev)
1487{
1488        enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1489        struct mlxsw_sp_ipip_entry *ipip_entry;
1490        enum mlxsw_sp_l3proto ul_proto;
1491        union mlxsw_sp_l3addr saddr;
1492        u32 ul_tb_id;
1493
1494        mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1495        if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1496                ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1497                ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1498                saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1499                if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1500                                                          saddr, ul_tb_id,
1501                                                          NULL)) {
1502                        ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1503                                                                ol_dev);
1504                        if (IS_ERR(ipip_entry))
1505                                return PTR_ERR(ipip_entry);
1506                }
1507        }
1508
1509        return 0;
1510}
1511
1512static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1513                                                   struct net_device *ol_dev)
1514{
1515        struct mlxsw_sp_ipip_entry *ipip_entry;
1516
1517        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1518        if (ipip_entry)
1519                mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1520}
1521
1522static void
1523mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1524                                struct mlxsw_sp_ipip_entry *ipip_entry)
1525{
1526        struct mlxsw_sp_fib_entry *decap_fib_entry;
1527
1528        decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1529        if (decap_fib_entry)
1530                mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1531                                                  decap_fib_entry);
1532}
1533
1534static int
1535mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1536                        u16 ul_rif_id, bool enable)
1537{
1538        struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1539        struct mlxsw_sp_rif *rif = &lb_rif->common;
1540        struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1541        char ritr_pl[MLXSW_REG_RITR_LEN];
1542        u32 saddr4;
1543
1544        switch (lb_cf.ul_protocol) {
1545        case MLXSW_SP_L3_PROTO_IPV4:
1546                saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1547                mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1548                                    rif->rif_index, rif->vr_id, rif->dev->mtu);
1549                mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1550                            MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1551                            ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1552                break;
1553
1554        case MLXSW_SP_L3_PROTO_IPV6:
1555                return -EAFNOSUPPORT;
1556        }
1557
1558        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1559}
1560
1561static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1562                                                 struct net_device *ol_dev)
1563{
1564        struct mlxsw_sp_ipip_entry *ipip_entry;
1565        struct mlxsw_sp_rif_ipip_lb *lb_rif;
1566        int err = 0;
1567
1568        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1569        if (ipip_entry) {
1570                lb_rif = ipip_entry->ol_lb;
1571                err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1572                                              lb_rif->ul_rif_id, true);
1573                if (err)
1574                        goto out;
1575                lb_rif->common.mtu = ol_dev->mtu;
1576        }
1577
1578out:
1579        return err;
1580}
1581
1582static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1583                                                struct net_device *ol_dev)
1584{
1585        struct mlxsw_sp_ipip_entry *ipip_entry;
1586
1587        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1588        if (ipip_entry)
1589                mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1590}
1591
1592static void
1593mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1594                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1595{
1596        if (ipip_entry->decap_fib_entry)
1597                mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1598}
1599
1600static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1601                                                  struct net_device *ol_dev)
1602{
1603        struct mlxsw_sp_ipip_entry *ipip_entry;
1604
1605        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1606        if (ipip_entry)
1607                mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1608}
1609
1610static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1611                                         struct mlxsw_sp_rif *old_rif,
1612                                         struct mlxsw_sp_rif *new_rif);
1613static int
1614mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1615                                 struct mlxsw_sp_ipip_entry *ipip_entry,
1616                                 bool keep_encap,
1617                                 struct netlink_ext_ack *extack)
1618{
1619        struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1620        struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1621
1622        new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1623                                                     ipip_entry->ipipt,
1624                                                     ipip_entry->ol_dev,
1625                                                     extack);
1626        if (IS_ERR(new_lb_rif))
1627                return PTR_ERR(new_lb_rif);
1628        ipip_entry->ol_lb = new_lb_rif;
1629
1630        if (keep_encap)
1631                mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1632                                             &new_lb_rif->common);
1633
1634        mlxsw_sp_rif_destroy(&old_lb_rif->common);
1635
1636        return 0;
1637}
1638
1639static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1640                                        struct mlxsw_sp_rif *rif);
1641
1642/**
1643 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1644 * @mlxsw_sp: mlxsw_sp.
1645 * @ipip_entry: IPIP entry.
1646 * @recreate_loopback: Recreates the associated loopback RIF.
1647 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1648 *              relevant when recreate_loopback is true.
1649 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1650 *                   is only relevant when recreate_loopback is false.
1651 * @extack: extack.
1652 *
1653 * Return: Non-zero value on failure.
1654 */
1655int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1656                                        struct mlxsw_sp_ipip_entry *ipip_entry,
1657                                        bool recreate_loopback,
1658                                        bool keep_encap,
1659                                        bool update_nexthops,
1660                                        struct netlink_ext_ack *extack)
1661{
1662        int err;
1663
1664        /* RIFs can't be edited, so to update loopback, we need to destroy and
1665         * recreate it. That creates a window of opportunity where RALUE and
1666         * RATR registers end up referencing a RIF that's already gone. RATRs
1667         * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1668         * of RALUE, demote the decap route back.
1669         */
1670        if (ipip_entry->decap_fib_entry)
1671                mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1672
1673        if (recreate_loopback) {
1674                err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1675                                                       keep_encap, extack);
1676                if (err)
1677                        return err;
1678        } else if (update_nexthops) {
1679                mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1680                                            &ipip_entry->ol_lb->common);
1681        }
1682
1683        if (ipip_entry->ol_dev->flags & IFF_UP)
1684                mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1685
1686        return 0;
1687}
1688
1689static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1690                                                struct net_device *ol_dev,
1691                                                struct netlink_ext_ack *extack)
1692{
1693        struct mlxsw_sp_ipip_entry *ipip_entry =
1694                mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1695
1696        if (!ipip_entry)
1697                return 0;
1698
1699        return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1700                                                   true, false, false, extack);
1701}
1702
1703static int
1704mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1705                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1706                                     struct net_device *ul_dev,
1707                                     bool *demote_this,
1708                                     struct netlink_ext_ack *extack)
1709{
1710        u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1711        enum mlxsw_sp_l3proto ul_proto;
1712        union mlxsw_sp_l3addr saddr;
1713
1714        /* Moving underlay to a different VRF might cause local address
1715         * conflict, and the conflicting tunnels need to be demoted.
1716         */
1717        ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1718        saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1719        if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1720                                                 saddr, ul_tb_id,
1721                                                 ipip_entry)) {
1722                *demote_this = true;
1723                return 0;
1724        }
1725
1726        return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1727                                                   true, true, false, extack);
1728}
1729
1730static int
1731mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1732                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1733                                    struct net_device *ul_dev)
1734{
1735        return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1736                                                   false, false, true, NULL);
1737}
1738
1739static int
1740mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1741                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1742                                      struct net_device *ul_dev)
1743{
1744        /* A down underlay device causes encapsulated packets to not be
1745         * forwarded, but decap still works. So refresh next hops without
1746         * touching anything else.
1747         */
1748        return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1749                                                   false, false, true, NULL);
1750}
1751
1752static int
1753mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1754                                        struct net_device *ol_dev,
1755                                        struct netlink_ext_ack *extack)
1756{
1757        const struct mlxsw_sp_ipip_ops *ipip_ops;
1758        struct mlxsw_sp_ipip_entry *ipip_entry;
1759        int err;
1760
1761        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1762        if (!ipip_entry)
1763                /* A change might make a tunnel eligible for offloading, but
1764                 * that is currently not implemented. What falls to slow path
1765                 * stays there.
1766                 */
1767                return 0;
1768
1769        /* A change might make a tunnel not eligible for offloading. */
1770        if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1771                                                 ipip_entry->ipipt)) {
1772                mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1773                return 0;
1774        }
1775
1776        ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1777        err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1778        return err;
1779}
1780
1781void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1782                                       struct mlxsw_sp_ipip_entry *ipip_entry)
1783{
1784        struct net_device *ol_dev = ipip_entry->ol_dev;
1785
1786        if (ol_dev->flags & IFF_UP)
1787                mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1788        mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1789}
1790
1791/* The configuration where several tunnels have the same local address in the
1792 * same underlay table needs special treatment in the HW. That is currently not
1793 * implemented in the driver. This function finds and demotes the first tunnel
1794 * with a given source address, except the one passed in in the argument
1795 * `except'.
1796 */
1797bool
1798mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1799                                     enum mlxsw_sp_l3proto ul_proto,
1800                                     union mlxsw_sp_l3addr saddr,
1801                                     u32 ul_tb_id,
1802                                     const struct mlxsw_sp_ipip_entry *except)
1803{
1804        struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1805
1806        list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1807                                 ipip_list_node) {
1808                if (ipip_entry != except &&
1809                    mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1810                                                      ul_tb_id, ipip_entry)) {
1811                        mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1812                        return true;
1813                }
1814        }
1815
1816        return false;
1817}
1818
1819static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1820                                                     struct net_device *ul_dev)
1821{
1822        struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1823
1824        list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1825                                 ipip_list_node) {
1826                struct net_device *ol_dev = ipip_entry->ol_dev;
1827                struct net_device *ipip_ul_dev;
1828
1829                rcu_read_lock();
1830                ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1831                rcu_read_unlock();
1832                if (ipip_ul_dev == ul_dev)
1833                        mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1834        }
1835}
1836
1837int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1838                                     struct net_device *ol_dev,
1839                                     unsigned long event,
1840                                     struct netdev_notifier_info *info)
1841{
1842        struct netdev_notifier_changeupper_info *chup;
1843        struct netlink_ext_ack *extack;
1844        int err = 0;
1845
1846        mutex_lock(&mlxsw_sp->router->lock);
1847        switch (event) {
1848        case NETDEV_REGISTER:
1849                err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1850                break;
1851        case NETDEV_UNREGISTER:
1852                mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1853                break;
1854        case NETDEV_UP:
1855                mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1856                break;
1857        case NETDEV_DOWN:
1858                mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1859                break;
1860        case NETDEV_CHANGEUPPER:
1861                chup = container_of(info, typeof(*chup), info);
1862                extack = info->extack;
1863                if (netif_is_l3_master(chup->upper_dev))
1864                        err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1865                                                                   ol_dev,
1866                                                                   extack);
1867                break;
1868        case NETDEV_CHANGE:
1869                extack = info->extack;
1870                err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1871                                                              ol_dev, extack);
1872                break;
1873        case NETDEV_CHANGEMTU:
1874                err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1875                break;
1876        }
1877        mutex_unlock(&mlxsw_sp->router->lock);
1878        return err;
1879}
1880
1881static int
1882__mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1883                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1884                                   struct net_device *ul_dev,
1885                                   bool *demote_this,
1886                                   unsigned long event,
1887                                   struct netdev_notifier_info *info)
1888{
1889        struct netdev_notifier_changeupper_info *chup;
1890        struct netlink_ext_ack *extack;
1891
1892        switch (event) {
1893        case NETDEV_CHANGEUPPER:
1894                chup = container_of(info, typeof(*chup), info);
1895                extack = info->extack;
1896                if (netif_is_l3_master(chup->upper_dev))
1897                        return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1898                                                                    ipip_entry,
1899                                                                    ul_dev,
1900                                                                    demote_this,
1901                                                                    extack);
1902                break;
1903
1904        case NETDEV_UP:
1905                return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1906                                                           ul_dev);
1907        case NETDEV_DOWN:
1908                return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1909                                                             ipip_entry,
1910                                                             ul_dev);
1911        }
1912        return 0;
1913}
1914
1915int
1916mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1917                                 struct net_device *ul_dev,
1918                                 unsigned long event,
1919                                 struct netdev_notifier_info *info)
1920{
1921        struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1922        int err = 0;
1923
1924        mutex_lock(&mlxsw_sp->router->lock);
1925        while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1926                                                                ul_dev,
1927                                                                ipip_entry))) {
1928                struct mlxsw_sp_ipip_entry *prev;
1929                bool demote_this = false;
1930
1931                err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1932                                                         ul_dev, &demote_this,
1933                                                         event, info);
1934                if (err) {
1935                        mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1936                                                                 ul_dev);
1937                        break;
1938                }
1939
1940                if (demote_this) {
1941                        if (list_is_first(&ipip_entry->ipip_list_node,
1942                                          &mlxsw_sp->router->ipip_list))
1943                                prev = NULL;
1944                        else
1945                                /* This can't be cached from previous iteration,
1946                                 * because that entry could be gone now.
1947                                 */
1948                                prev = list_prev_entry(ipip_entry,
1949                                                       ipip_list_node);
1950                        mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1951                        ipip_entry = prev;
1952                }
1953        }
1954        mutex_unlock(&mlxsw_sp->router->lock);
1955
1956        return err;
1957}
1958
1959int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1960                                      enum mlxsw_sp_l3proto ul_proto,
1961                                      const union mlxsw_sp_l3addr *ul_sip,
1962                                      u32 tunnel_index)
1963{
1964        enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1965        struct mlxsw_sp_router *router = mlxsw_sp->router;
1966        struct mlxsw_sp_fib_entry *fib_entry;
1967        int err = 0;
1968
1969        mutex_lock(&mlxsw_sp->router->lock);
1970
1971        if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
1972                err = -EINVAL;
1973                goto out;
1974        }
1975
1976        router->nve_decap_config.ul_tb_id = ul_tb_id;
1977        router->nve_decap_config.tunnel_index = tunnel_index;
1978        router->nve_decap_config.ul_proto = ul_proto;
1979        router->nve_decap_config.ul_sip = *ul_sip;
1980        router->nve_decap_config.valid = true;
1981
1982        /* It is valid to create a tunnel with a local IP and only later
1983         * assign this IP address to a local interface
1984         */
1985        fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1986                                                         ul_proto, ul_sip,
1987                                                         type);
1988        if (!fib_entry)
1989                goto out;
1990
1991        fib_entry->decap.tunnel_index = tunnel_index;
1992        fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1993
1994        err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1995        if (err)
1996                goto err_fib_entry_update;
1997
1998        goto out;
1999
2000err_fib_entry_update:
2001        fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2002        mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2003out:
2004        mutex_unlock(&mlxsw_sp->router->lock);
2005        return err;
2006}
2007
2008void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2009                                      enum mlxsw_sp_l3proto ul_proto,
2010                                      const union mlxsw_sp_l3addr *ul_sip)
2011{
2012        enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2013        struct mlxsw_sp_router *router = mlxsw_sp->router;
2014        struct mlxsw_sp_fib_entry *fib_entry;
2015
2016        mutex_lock(&mlxsw_sp->router->lock);
2017
2018        if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2019                goto out;
2020
2021        router->nve_decap_config.valid = false;
2022
2023        fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2024                                                         ul_proto, ul_sip,
2025                                                         type);
2026        if (!fib_entry)
2027                goto out;
2028
2029        fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2030        mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2031out:
2032        mutex_unlock(&mlxsw_sp->router->lock);
2033}
2034
2035static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2036                                         u32 ul_tb_id,
2037                                         enum mlxsw_sp_l3proto ul_proto,
2038                                         const union mlxsw_sp_l3addr *ul_sip)
2039{
2040        struct mlxsw_sp_router *router = mlxsw_sp->router;
2041
2042        return router->nve_decap_config.valid &&
2043               router->nve_decap_config.ul_tb_id == ul_tb_id &&
2044               router->nve_decap_config.ul_proto == ul_proto &&
2045               !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2046                       sizeof(*ul_sip));
2047}
2048
2049struct mlxsw_sp_neigh_key {
2050        struct neighbour *n;
2051};
2052
2053struct mlxsw_sp_neigh_entry {
2054        struct list_head rif_list_node;
2055        struct rhash_head ht_node;
2056        struct mlxsw_sp_neigh_key key;
2057        u16 rif;
2058        bool connected;
2059        unsigned char ha[ETH_ALEN];
2060        struct list_head nexthop_list; /* list of nexthops using
2061                                        * this neigh entry
2062                                        */
2063        struct list_head nexthop_neighs_list_node;
2064        unsigned int counter_index;
2065        bool counter_valid;
2066};
2067
2068static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2069        .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2070        .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2071        .key_len = sizeof(struct mlxsw_sp_neigh_key),
2072};
2073
2074struct mlxsw_sp_neigh_entry *
2075mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2076                        struct mlxsw_sp_neigh_entry *neigh_entry)
2077{
2078        if (!neigh_entry) {
2079                if (list_empty(&rif->neigh_list))
2080                        return NULL;
2081                else
2082                        return list_first_entry(&rif->neigh_list,
2083                                                typeof(*neigh_entry),
2084                                                rif_list_node);
2085        }
2086        if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2087                return NULL;
2088        return list_next_entry(neigh_entry, rif_list_node);
2089}
2090
2091int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2092{
2093        return neigh_entry->key.n->tbl->family;
2094}
2095
2096unsigned char *
2097mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2098{
2099        return neigh_entry->ha;
2100}
2101
2102u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2103{
2104        struct neighbour *n;
2105
2106        n = neigh_entry->key.n;
2107        return ntohl(*((__be32 *) n->primary_key));
2108}
2109
2110struct in6_addr *
2111mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2112{
2113        struct neighbour *n;
2114
2115        n = neigh_entry->key.n;
2116        return (struct in6_addr *) &n->primary_key;
2117}
2118
2119int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2120                               struct mlxsw_sp_neigh_entry *neigh_entry,
2121                               u64 *p_counter)
2122{
2123        if (!neigh_entry->counter_valid)
2124                return -EINVAL;
2125
2126        return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2127                                         p_counter, NULL);
2128}
2129
2130static struct mlxsw_sp_neigh_entry *
2131mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2132                           u16 rif)
2133{
2134        struct mlxsw_sp_neigh_entry *neigh_entry;
2135
2136        neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2137        if (!neigh_entry)
2138                return NULL;
2139
2140        neigh_entry->key.n = n;
2141        neigh_entry->rif = rif;
2142        INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2143
2144        return neigh_entry;
2145}
2146
2147static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2148{
2149        kfree(neigh_entry);
2150}
2151
2152static int
2153mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2154                            struct mlxsw_sp_neigh_entry *neigh_entry)
2155{
2156        return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2157                                      &neigh_entry->ht_node,
2158                                      mlxsw_sp_neigh_ht_params);
2159}
2160
2161static void
2162mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2163                            struct mlxsw_sp_neigh_entry *neigh_entry)
2164{
2165        rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2166                               &neigh_entry->ht_node,
2167                               mlxsw_sp_neigh_ht_params);
2168}
2169
2170static bool
2171mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2172                                    struct mlxsw_sp_neigh_entry *neigh_entry)
2173{
2174        struct devlink *devlink;
2175        const char *table_name;
2176
2177        switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2178        case AF_INET:
2179                table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2180                break;
2181        case AF_INET6:
2182                table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2183                break;
2184        default:
2185                WARN_ON(1);
2186                return false;
2187        }
2188
2189        devlink = priv_to_devlink(mlxsw_sp->core);
2190        return devlink_dpipe_table_counter_enabled(devlink, table_name);
2191}
2192
2193static void
2194mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2195                             struct mlxsw_sp_neigh_entry *neigh_entry)
2196{
2197        if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2198                return;
2199
2200        if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2201                return;
2202
2203        neigh_entry->counter_valid = true;
2204}
2205
2206static void
2207mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2208                            struct mlxsw_sp_neigh_entry *neigh_entry)
2209{
2210        if (!neigh_entry->counter_valid)
2211                return;
2212        mlxsw_sp_flow_counter_free(mlxsw_sp,
2213                                   neigh_entry->counter_index);
2214        neigh_entry->counter_valid = false;
2215}
2216
2217static struct mlxsw_sp_neigh_entry *
2218mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2219{
2220        struct mlxsw_sp_neigh_entry *neigh_entry;
2221        struct mlxsw_sp_rif *rif;
2222        int err;
2223
2224        rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2225        if (!rif)
2226                return ERR_PTR(-EINVAL);
2227
2228        neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2229        if (!neigh_entry)
2230                return ERR_PTR(-ENOMEM);
2231
2232        err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2233        if (err)
2234                goto err_neigh_entry_insert;
2235
2236        mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2237        list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2238
2239        return neigh_entry;
2240
2241err_neigh_entry_insert:
2242        mlxsw_sp_neigh_entry_free(neigh_entry);
2243        return ERR_PTR(err);
2244}
2245
2246static void
2247mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2248                             struct mlxsw_sp_neigh_entry *neigh_entry)
2249{
2250        list_del(&neigh_entry->rif_list_node);
2251        mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2252        mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2253        mlxsw_sp_neigh_entry_free(neigh_entry);
2254}
2255
2256static struct mlxsw_sp_neigh_entry *
2257mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2258{
2259        struct mlxsw_sp_neigh_key key;
2260
2261        key.n = n;
2262        return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2263                                      &key, mlxsw_sp_neigh_ht_params);
2264}
2265
2266static void
2267mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2268{
2269        unsigned long interval;
2270
2271#if IS_ENABLED(CONFIG_IPV6)
2272        interval = min_t(unsigned long,
2273                         NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2274                         NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2275#else
2276        interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2277#endif
2278        mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2279}
2280
2281static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2282                                                   char *rauhtd_pl,
2283                                                   int ent_index)
2284{
2285        u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2286        struct net_device *dev;
2287        struct neighbour *n;
2288        __be32 dipn;
2289        u32 dip;
2290        u16 rif;
2291
2292        mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2293
2294        if (WARN_ON_ONCE(rif >= max_rifs))
2295                return;
2296        if (!mlxsw_sp->router->rifs[rif]) {
2297                dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2298                return;
2299        }
2300
2301        dipn = htonl(dip);
2302        dev = mlxsw_sp->router->rifs[rif]->dev;
2303        n = neigh_lookup(&arp_tbl, &dipn, dev);
2304        if (!n)
2305                return;
2306
2307        netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2308        neigh_event_send(n, NULL);
2309        neigh_release(n);
2310}
2311
2312#if IS_ENABLED(CONFIG_IPV6)
2313static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2314                                                   char *rauhtd_pl,
2315                                                   int rec_index)
2316{
2317        struct net_device *dev;
2318        struct neighbour *n;
2319        struct in6_addr dip;
2320        u16 rif;
2321
2322        mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2323                                         (char *) &dip);
2324
2325        if (!mlxsw_sp->router->rifs[rif]) {
2326                dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2327                return;
2328        }
2329
2330        dev = mlxsw_sp->router->rifs[rif]->dev;
2331        n = neigh_lookup(&nd_tbl, &dip, dev);
2332        if (!n)
2333                return;
2334
2335        netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2336        neigh_event_send(n, NULL);
2337        neigh_release(n);
2338}
2339#else
2340static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2341                                                   char *rauhtd_pl,
2342                                                   int rec_index)
2343{
2344}
2345#endif
2346
2347static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2348                                                   char *rauhtd_pl,
2349                                                   int rec_index)
2350{
2351        u8 num_entries;
2352        int i;
2353
2354        num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2355                                                                rec_index);
2356        /* Hardware starts counting at 0, so add 1. */
2357        num_entries++;
2358
2359        /* Each record consists of several neighbour entries. */
2360        for (i = 0; i < num_entries; i++) {
2361                int ent_index;
2362
2363                ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2364                mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2365                                                       ent_index);
2366        }
2367
2368}
2369
2370static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2371                                                   char *rauhtd_pl,
2372                                                   int rec_index)
2373{
2374        /* One record contains one entry. */
2375        mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2376                                               rec_index);
2377}
2378
2379static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2380                                              char *rauhtd_pl, int rec_index)
2381{
2382        switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2383        case MLXSW_REG_RAUHTD_TYPE_IPV4:
2384                mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2385                                                       rec_index);
2386                break;
2387        case MLXSW_REG_RAUHTD_TYPE_IPV6:
2388                mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2389                                                       rec_index);
2390                break;
2391        }
2392}
2393
2394static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2395{
2396        u8 num_rec, last_rec_index, num_entries;
2397
2398        num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2399        last_rec_index = num_rec - 1;
2400
2401        if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2402                return false;
2403        if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2404            MLXSW_REG_RAUHTD_TYPE_IPV6)
2405                return true;
2406
2407        num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2408                                                                last_rec_index);
2409        if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2410                return true;
2411        return false;
2412}
2413
2414static int
2415__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2416                                       char *rauhtd_pl,
2417                                       enum mlxsw_reg_rauhtd_type type)
2418{
2419        int i, num_rec;
2420        int err;
2421
2422        /* Ensure the RIF we read from the device does not change mid-dump. */
2423        mutex_lock(&mlxsw_sp->router->lock);
2424        do {
2425                mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2426                err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2427                                      rauhtd_pl);
2428                if (err) {
2429                        dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2430                        break;
2431                }
2432                num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2433                for (i = 0; i < num_rec; i++)
2434                        mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2435                                                          i);
2436        } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2437        mutex_unlock(&mlxsw_sp->router->lock);
2438
2439        return err;
2440}
2441
2442static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2443{
2444        enum mlxsw_reg_rauhtd_type type;
2445        char *rauhtd_pl;
2446        int err;
2447
2448        rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2449        if (!rauhtd_pl)
2450                return -ENOMEM;
2451
2452        type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2453        err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2454        if (err)
2455                goto out;
2456
2457        type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2458        err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2459out:
2460        kfree(rauhtd_pl);
2461        return err;
2462}
2463
2464static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2465{
2466        struct mlxsw_sp_neigh_entry *neigh_entry;
2467
2468        mutex_lock(&mlxsw_sp->router->lock);
2469        list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2470                            nexthop_neighs_list_node)
2471                /* If this neigh have nexthops, make the kernel think this neigh
2472                 * is active regardless of the traffic.
2473                 */
2474                neigh_event_send(neigh_entry->key.n, NULL);
2475        mutex_unlock(&mlxsw_sp->router->lock);
2476}
2477
2478static void
2479mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2480{
2481        unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2482
2483        mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2484                               msecs_to_jiffies(interval));
2485}
2486
2487static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2488{
2489        struct mlxsw_sp_router *router;
2490        int err;
2491
2492        router = container_of(work, struct mlxsw_sp_router,
2493                              neighs_update.dw.work);
2494        err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2495        if (err)
2496                dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2497
2498        mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2499
2500        mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2501}
2502
2503static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2504{
2505        struct mlxsw_sp_neigh_entry *neigh_entry;
2506        struct mlxsw_sp_router *router;
2507
2508        router = container_of(work, struct mlxsw_sp_router,
2509                              nexthop_probe_dw.work);
2510        /* Iterate over nexthop neighbours, find those who are unresolved and
2511         * send arp on them. This solves the chicken-egg problem when
2512         * the nexthop wouldn't get offloaded until the neighbor is resolved
2513         * but it wouldn't get resolved ever in case traffic is flowing in HW
2514         * using different nexthop.
2515         */
2516        mutex_lock(&router->lock);
2517        list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2518                            nexthop_neighs_list_node)
2519                if (!neigh_entry->connected)
2520                        neigh_event_send(neigh_entry->key.n, NULL);
2521        mutex_unlock(&router->lock);
2522
2523        mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2524                               MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2525}
2526
2527static void
2528mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2529                              struct mlxsw_sp_neigh_entry *neigh_entry,
2530                              bool removing, bool dead);
2531
2532static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2533{
2534        return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2535                        MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2536}
2537
2538static int
2539mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2540                                struct mlxsw_sp_neigh_entry *neigh_entry,
2541                                enum mlxsw_reg_rauht_op op)
2542{
2543        struct neighbour *n = neigh_entry->key.n;
2544        u32 dip = ntohl(*((__be32 *) n->primary_key));
2545        char rauht_pl[MLXSW_REG_RAUHT_LEN];
2546
2547        mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2548                              dip);
2549        if (neigh_entry->counter_valid)
2550                mlxsw_reg_rauht_pack_counter(rauht_pl,
2551                                             neigh_entry->counter_index);
2552        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2553}
2554
2555static int
2556mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2557                                struct mlxsw_sp_neigh_entry *neigh_entry,
2558                                enum mlxsw_reg_rauht_op op)
2559{
2560        struct neighbour *n = neigh_entry->key.n;
2561        char rauht_pl[MLXSW_REG_RAUHT_LEN];
2562        const char *dip = n->primary_key;
2563
2564        mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2565                              dip);
2566        if (neigh_entry->counter_valid)
2567                mlxsw_reg_rauht_pack_counter(rauht_pl,
2568                                             neigh_entry->counter_index);
2569        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2570}
2571
2572bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2573{
2574        struct neighbour *n = neigh_entry->key.n;
2575
2576        /* Packets with a link-local destination address are trapped
2577         * after LPM lookup and never reach the neighbour table, so
2578         * there is no need to program such neighbours to the device.
2579         */
2580        if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2581            IPV6_ADDR_LINKLOCAL)
2582                return true;
2583        return false;
2584}
2585
2586static void
2587mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2588                            struct mlxsw_sp_neigh_entry *neigh_entry,
2589                            bool adding)
2590{
2591        enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2592        int err;
2593
2594        if (!adding && !neigh_entry->connected)
2595                return;
2596        neigh_entry->connected = adding;
2597        if (neigh_entry->key.n->tbl->family == AF_INET) {
2598                err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2599                                                      op);
2600                if (err)
2601                        return;
2602        } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2603                if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2604                        return;
2605                err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2606                                                      op);
2607                if (err)
2608                        return;
2609        } else {
2610                WARN_ON_ONCE(1);
2611                return;
2612        }
2613
2614        if (adding)
2615                neigh_entry->key.n->flags |= NTF_OFFLOADED;
2616        else
2617                neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2618}
2619
2620void
2621mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2622                                    struct mlxsw_sp_neigh_entry *neigh_entry,
2623                                    bool adding)
2624{
2625        if (adding)
2626                mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2627        else
2628                mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2629        mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2630}
2631
2632struct mlxsw_sp_netevent_work {
2633        struct work_struct work;
2634        struct mlxsw_sp *mlxsw_sp;
2635        struct neighbour *n;
2636};
2637
2638static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2639{
2640        struct mlxsw_sp_netevent_work *net_work =
2641                container_of(work, struct mlxsw_sp_netevent_work, work);
2642        struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2643        struct mlxsw_sp_neigh_entry *neigh_entry;
2644        struct neighbour *n = net_work->n;
2645        unsigned char ha[ETH_ALEN];
2646        bool entry_connected;
2647        u8 nud_state, dead;
2648
2649        /* If these parameters are changed after we release the lock,
2650         * then we are guaranteed to receive another event letting us
2651         * know about it.
2652         */
2653        read_lock_bh(&n->lock);
2654        memcpy(ha, n->ha, ETH_ALEN);
2655        nud_state = n->nud_state;
2656        dead = n->dead;
2657        read_unlock_bh(&n->lock);
2658
2659        mutex_lock(&mlxsw_sp->router->lock);
2660        mlxsw_sp_span_respin(mlxsw_sp);
2661
2662        entry_connected = nud_state & NUD_VALID && !dead;
2663        neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2664        if (!entry_connected && !neigh_entry)
2665                goto out;
2666        if (!neigh_entry) {
2667                neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2668                if (IS_ERR(neigh_entry))
2669                        goto out;
2670        }
2671
2672        if (neigh_entry->connected && entry_connected &&
2673            !memcmp(neigh_entry->ha, ha, ETH_ALEN))
2674                goto out;
2675
2676        memcpy(neigh_entry->ha, ha, ETH_ALEN);
2677        mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2678        mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2679                                      dead);
2680
2681        if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2682                mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2683
2684out:
2685        mutex_unlock(&mlxsw_sp->router->lock);
2686        neigh_release(n);
2687        kfree(net_work);
2688}
2689
2690static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2691
2692static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2693{
2694        struct mlxsw_sp_netevent_work *net_work =
2695                container_of(work, struct mlxsw_sp_netevent_work, work);
2696        struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2697
2698        mlxsw_sp_mp_hash_init(mlxsw_sp);
2699        kfree(net_work);
2700}
2701
2702static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2703
2704static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2705{
2706        struct mlxsw_sp_netevent_work *net_work =
2707                container_of(work, struct mlxsw_sp_netevent_work, work);
2708        struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2709
2710        __mlxsw_sp_router_init(mlxsw_sp);
2711        kfree(net_work);
2712}
2713
2714static int mlxsw_sp_router_schedule_work(struct net *net,
2715                                         struct notifier_block *nb,
2716                                         void (*cb)(struct work_struct *))
2717{
2718        struct mlxsw_sp_netevent_work *net_work;
2719        struct mlxsw_sp_router *router;
2720
2721        router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2722        if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2723                return NOTIFY_DONE;
2724
2725        net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2726        if (!net_work)
2727                return NOTIFY_BAD;
2728
2729        INIT_WORK(&net_work->work, cb);
2730        net_work->mlxsw_sp = router->mlxsw_sp;
2731        mlxsw_core_schedule_work(&net_work->work);
2732        return NOTIFY_DONE;
2733}
2734
2735static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2736                                          unsigned long event, void *ptr)
2737{
2738        struct mlxsw_sp_netevent_work *net_work;
2739        struct mlxsw_sp_port *mlxsw_sp_port;
2740        struct mlxsw_sp *mlxsw_sp;
2741        unsigned long interval;
2742        struct neigh_parms *p;
2743        struct neighbour *n;
2744
2745        switch (event) {
2746        case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2747                p = ptr;
2748
2749                /* We don't care about changes in the default table. */
2750                if (!p->dev || (p->tbl->family != AF_INET &&
2751                                p->tbl->family != AF_INET6))
2752                        return NOTIFY_DONE;
2753
2754                /* We are in atomic context and can't take RTNL mutex,
2755                 * so use RCU variant to walk the device chain.
2756                 */
2757                mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2758                if (!mlxsw_sp_port)
2759                        return NOTIFY_DONE;
2760
2761                mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2762                interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2763                mlxsw_sp->router->neighs_update.interval = interval;
2764
2765                mlxsw_sp_port_dev_put(mlxsw_sp_port);
2766                break;
2767        case NETEVENT_NEIGH_UPDATE:
2768                n = ptr;
2769
2770                if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2771                        return NOTIFY_DONE;
2772
2773                mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2774                if (!mlxsw_sp_port)
2775                        return NOTIFY_DONE;
2776
2777                net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2778                if (!net_work) {
2779                        mlxsw_sp_port_dev_put(mlxsw_sp_port);
2780                        return NOTIFY_BAD;
2781                }
2782
2783                INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2784                net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2785                net_work->n = n;
2786
2787                /* Take a reference to ensure the neighbour won't be
2788                 * destructed until we drop the reference in delayed
2789                 * work.
2790                 */
2791                neigh_clone(n);
2792                mlxsw_core_schedule_work(&net_work->work);
2793                mlxsw_sp_port_dev_put(mlxsw_sp_port);
2794                break;
2795        case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2796        case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2797                return mlxsw_sp_router_schedule_work(ptr, nb,
2798                                mlxsw_sp_router_mp_hash_event_work);
2799
2800        case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2801                return mlxsw_sp_router_schedule_work(ptr, nb,
2802                                mlxsw_sp_router_update_priority_work);
2803        }
2804
2805        return NOTIFY_DONE;
2806}
2807
2808static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2809{
2810        int err;
2811
2812        err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2813                              &mlxsw_sp_neigh_ht_params);
2814        if (err)
2815                return err;
2816
2817        /* Initialize the polling interval according to the default
2818         * table.
2819         */
2820        mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2821
2822        /* Create the delayed works for the activity_update */
2823        INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2824                          mlxsw_sp_router_neighs_update_work);
2825        INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2826                          mlxsw_sp_router_probe_unresolved_nexthops);
2827        mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2828        mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2829        return 0;
2830}
2831
2832static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2833{
2834        cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2835        cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2836        rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2837}
2838
2839static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2840                                         struct mlxsw_sp_rif *rif)
2841{
2842        struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2843
2844        list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2845                                 rif_list_node) {
2846                mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2847                mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2848        }
2849}
2850
2851enum mlxsw_sp_nexthop_type {
2852        MLXSW_SP_NEXTHOP_TYPE_ETH,
2853        MLXSW_SP_NEXTHOP_TYPE_IPIP,
2854};
2855
2856enum mlxsw_sp_nexthop_action {
2857        /* Nexthop forwards packets to an egress RIF */
2858        MLXSW_SP_NEXTHOP_ACTION_FORWARD,
2859        /* Nexthop discards packets */
2860        MLXSW_SP_NEXTHOP_ACTION_DISCARD,
2861        /* Nexthop traps packets */
2862        MLXSW_SP_NEXTHOP_ACTION_TRAP,
2863};
2864
2865struct mlxsw_sp_nexthop_key {
2866        struct fib_nh *fib_nh;
2867};
2868
2869struct mlxsw_sp_nexthop {
2870        struct list_head neigh_list_node; /* member of neigh entry list */
2871        struct list_head rif_list_node;
2872        struct list_head router_list_node;
2873        struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2874                                                   * this nexthop belongs to
2875                                                   */
2876        struct rhash_head ht_node;
2877        struct neigh_table *neigh_tbl;
2878        struct mlxsw_sp_nexthop_key key;
2879        unsigned char gw_addr[sizeof(struct in6_addr)];
2880        int ifindex;
2881        int nh_weight;
2882        int norm_nh_weight;
2883        int num_adj_entries;
2884        struct mlxsw_sp_rif *rif;
2885        u8 should_offload:1, /* set indicates this nexthop should be written
2886                              * to the adjacency table.
2887                              */
2888           offloaded:1, /* set indicates this nexthop was written to the
2889                         * adjacency table.
2890                         */
2891           update:1; /* set indicates this nexthop should be updated in the
2892                      * adjacency table (f.e., its MAC changed).
2893                      */
2894        enum mlxsw_sp_nexthop_action action;
2895        enum mlxsw_sp_nexthop_type type;
2896        union {
2897                struct mlxsw_sp_neigh_entry *neigh_entry;
2898                struct mlxsw_sp_ipip_entry *ipip_entry;
2899        };
2900        unsigned int counter_index;
2901        bool counter_valid;
2902};
2903
2904enum mlxsw_sp_nexthop_group_type {
2905        MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2906        MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2907        MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
2908};
2909
2910struct mlxsw_sp_nexthop_group_info {
2911        struct mlxsw_sp_nexthop_group *nh_grp;
2912        u32 adj_index;
2913        u16 ecmp_size;
2914        u16 count;
2915        int sum_norm_weight;
2916        u8 adj_index_valid:1,
2917           gateway:1, /* routes using the group use a gateway */
2918           is_resilient:1;
2919        struct list_head list; /* member in nh_res_grp_list */
2920        struct mlxsw_sp_nexthop nexthops[0];
2921#define nh_rif  nexthops[0].rif
2922};
2923
2924struct mlxsw_sp_nexthop_group_vr_key {
2925        u16 vr_id;
2926        enum mlxsw_sp_l3proto proto;
2927};
2928
2929struct mlxsw_sp_nexthop_group_vr_entry {
2930        struct list_head list; /* member in vr_list */
2931        struct rhash_head ht_node; /* member in vr_ht */
2932        refcount_t ref_count;
2933        struct mlxsw_sp_nexthop_group_vr_key key;
2934};
2935
2936struct mlxsw_sp_nexthop_group {
2937        struct rhash_head ht_node;
2938        struct list_head fib_list; /* list of fib entries that use this group */
2939        union {
2940                struct {
2941                        struct fib_info *fi;
2942                } ipv4;
2943                struct {
2944                        u32 id;
2945                } obj;
2946        };
2947        struct mlxsw_sp_nexthop_group_info *nhgi;
2948        struct list_head vr_list;
2949        struct rhashtable vr_ht;
2950        enum mlxsw_sp_nexthop_group_type type;
2951        bool can_destroy;
2952};
2953
2954void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2955                                    struct mlxsw_sp_nexthop *nh)
2956{
2957        struct devlink *devlink;
2958
2959        devlink = priv_to_devlink(mlxsw_sp->core);
2960        if (!devlink_dpipe_table_counter_enabled(devlink,
2961                                                 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2962                return;
2963
2964        if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2965                return;
2966
2967        nh->counter_valid = true;
2968}
2969
2970void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2971                                   struct mlxsw_sp_nexthop *nh)
2972{
2973        if (!nh->counter_valid)
2974                return;
2975        mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2976        nh->counter_valid = false;
2977}
2978
2979int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2980                                 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2981{
2982        if (!nh->counter_valid)
2983                return -EINVAL;
2984
2985        return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2986                                         p_counter, NULL);
2987}
2988
2989struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2990                                               struct mlxsw_sp_nexthop *nh)
2991{
2992        if (!nh) {
2993                if (list_empty(&router->nexthop_list))
2994                        return NULL;
2995                else
2996                        return list_first_entry(&router->nexthop_list,
2997                                                typeof(*nh), router_list_node);
2998        }
2999        if (list_is_last(&nh->router_list_node, &router->nexthop_list))
3000                return NULL;
3001        return list_next_entry(nh, router_list_node);
3002}
3003
3004bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
3005{
3006        return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3007}
3008
3009unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3010{
3011        if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3012            !mlxsw_sp_nexthop_is_forward(nh))
3013                return NULL;
3014        return nh->neigh_entry->ha;
3015}
3016
3017int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3018                             u32 *p_adj_size, u32 *p_adj_hash_index)
3019{
3020        struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3021        u32 adj_hash_index = 0;
3022        int i;
3023
3024        if (!nh->offloaded || !nhgi->adj_index_valid)
3025                return -EINVAL;
3026
3027        *p_adj_index = nhgi->adj_index;
3028        *p_adj_size = nhgi->ecmp_size;
3029
3030        for (i = 0; i < nhgi->count; i++) {
3031                struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3032
3033                if (nh_iter == nh)
3034                        break;
3035                if (nh_iter->offloaded)
3036                        adj_hash_index += nh_iter->num_adj_entries;
3037        }
3038
3039        *p_adj_hash_index = adj_hash_index;
3040        return 0;
3041}
3042
3043struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3044{
3045        return nh->rif;
3046}
3047
3048bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3049{
3050        struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3051        int i;
3052
3053        for (i = 0; i < nhgi->count; i++) {
3054                struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3055
3056                if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3057                        return true;
3058        }
3059        return false;
3060}
3061
3062static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3063        .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3064        .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3065        .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3066        .automatic_shrinking = true,
3067};
3068
3069static struct mlxsw_sp_nexthop_group_vr_entry *
3070mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3071                                       const struct mlxsw_sp_fib *fib)
3072{
3073        struct mlxsw_sp_nexthop_group_vr_key key;
3074
3075        memset(&key, 0, sizeof(key));
3076        key.vr_id = fib->vr->id;
3077        key.proto = fib->proto;
3078        return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
3079                                      mlxsw_sp_nexthop_group_vr_ht_params);
3080}
3081
3082static int
3083mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3084                                       const struct mlxsw_sp_fib *fib)
3085{
3086        struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3087        int err;
3088
3089        vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
3090        if (!vr_entry)
3091                return -ENOMEM;
3092
3093        vr_entry->key.vr_id = fib->vr->id;
3094        vr_entry->key.proto = fib->proto;
3095        refcount_set(&vr_entry->ref_count, 1);
3096
3097        err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3098                                     mlxsw_sp_nexthop_group_vr_ht_params);
3099        if (err)
3100                goto err_hashtable_insert;
3101
3102        list_add(&vr_entry->list, &nh_grp->vr_list);
3103
3104        return 0;
3105
3106err_hashtable_insert:
3107        kfree(vr_entry);
3108        return err;
3109}
3110
3111static void
3112mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3113                                        struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3114{
3115        list_del(&vr_entry->list);
3116        rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3117                               mlxsw_sp_nexthop_group_vr_ht_params);
3118        kfree(vr_entry);
3119}
3120
3121static int
3122mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3123                               const struct mlxsw_sp_fib *fib)
3124{
3125        struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3126
3127        vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3128        if (vr_entry) {
3129                refcount_inc(&vr_entry->ref_count);
3130                return 0;
3131        }
3132
3133        return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3134}
3135
3136static void
3137mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3138                                 const struct mlxsw_sp_fib *fib)
3139{
3140        struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3141
3142        vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3143        if (WARN_ON_ONCE(!vr_entry))
3144                return;
3145
3146        if (!refcount_dec_and_test(&vr_entry->ref_count))
3147                return;
3148
3149        mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3150}
3151
3152struct mlxsw_sp_nexthop_group_cmp_arg {
3153        enum mlxsw_sp_nexthop_group_type type;
3154        union {
3155                struct fib_info *fi;
3156                struct mlxsw_sp_fib6_entry *fib6_entry;
3157                u32 id;
3158        };
3159};
3160
3161static bool
3162mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3163                                    const struct in6_addr *gw, int ifindex,
3164                                    int weight)
3165{
3166        int i;
3167
3168        for (i = 0; i < nh_grp->nhgi->count; i++) {
3169                const struct mlxsw_sp_nexthop *nh;
3170
3171                nh = &nh_grp->nhgi->nexthops[i];
3172                if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3173                    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3174                        return true;
3175        }
3176
3177        return false;
3178}
3179
3180static bool
3181mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3182                            const struct mlxsw_sp_fib6_entry *fib6_entry)
3183{
3184        struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3185
3186        if (nh_grp->nhgi->count != fib6_entry->nrt6)
3187                return false;
3188
3189        list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3190                struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3191                struct in6_addr *gw;
3192                int ifindex, weight;
3193
3194                ifindex = fib6_nh->fib_nh_dev->ifindex;
3195                weight = fib6_nh->fib_nh_weight;
3196                gw = &fib6_nh->fib_nh_gw6;
3197                if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3198                                                         weight))
3199                        return false;
3200        }
3201
3202        return true;
3203}
3204
3205static int
3206mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3207{
3208        const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3209        const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3210
3211        if (nh_grp->type != cmp_arg->type)
3212                return 1;
3213
3214        switch (cmp_arg->type) {
3215        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3216                return cmp_arg->fi != nh_grp->ipv4.fi;
3217        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3218                return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3219                                                    cmp_arg->fib6_entry);
3220        case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3221                return cmp_arg->id != nh_grp->obj.id;
3222        default:
3223                WARN_ON(1);
3224                return 1;
3225        }
3226}
3227
3228static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3229{
3230        const struct mlxsw_sp_nexthop_group *nh_grp = data;
3231        const struct mlxsw_sp_nexthop *nh;
3232        struct fib_info *fi;
3233        unsigned int val;
3234        int i;
3235
3236        switch (nh_grp->type) {
3237        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3238                fi = nh_grp->ipv4.fi;
3239                return jhash(&fi, sizeof(fi), seed);
3240        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3241                val = nh_grp->nhgi->count;
3242                for (i = 0; i < nh_grp->nhgi->count; i++) {
3243                        nh = &nh_grp->nhgi->nexthops[i];
3244                        val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3245                        val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3246                }
3247                return jhash(&val, sizeof(val), seed);
3248        case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3249                return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
3250        default:
3251                WARN_ON(1);
3252                return 0;
3253        }
3254}
3255
3256static u32
3257mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3258{
3259        unsigned int val = fib6_entry->nrt6;
3260        struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3261
3262        list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3263                struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3264                struct net_device *dev = fib6_nh->fib_nh_dev;
3265                struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3266
3267                val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3268                val ^= jhash(gw, sizeof(*gw), seed);
3269        }
3270
3271        return jhash(&val, sizeof(val), seed);
3272}
3273
3274static u32
3275mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3276{
3277        const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3278
3279        switch (cmp_arg->type) {
3280        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3281                return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3282        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3283                return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3284        case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3285                return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
3286        default:
3287                WARN_ON(1);
3288                return 0;
3289        }
3290}
3291
3292static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3293        .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3294        .hashfn      = mlxsw_sp_nexthop_group_hash,
3295        .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3296        .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3297};
3298
3299static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3300                                         struct mlxsw_sp_nexthop_group *nh_grp)
3301{
3302        if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3303            !nh_grp->nhgi->gateway)
3304                return 0;
3305
3306        return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3307                                      &nh_grp->ht_node,
3308                                      mlxsw_sp_nexthop_group_ht_params);
3309}
3310
3311static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3312                                          struct mlxsw_sp_nexthop_group *nh_grp)
3313{
3314        if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3315            !nh_grp->nhgi->gateway)
3316                return;
3317
3318        rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3319                               &nh_grp->ht_node,
3320                               mlxsw_sp_nexthop_group_ht_params);
3321}
3322
3323static struct mlxsw_sp_nexthop_group *
3324mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3325                               struct fib_info *fi)
3326{
3327        struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3328
3329        cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3330        cmp_arg.fi = fi;
3331        return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3332                                      &cmp_arg,
3333                                      mlxsw_sp_nexthop_group_ht_params);
3334}
3335
3336static struct mlxsw_sp_nexthop_group *
3337mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3338                               struct mlxsw_sp_fib6_entry *fib6_entry)
3339{
3340        struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3341
3342        cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3343        cmp_arg.fib6_entry = fib6_entry;
3344        return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3345                                      &cmp_arg,
3346                                      mlxsw_sp_nexthop_group_ht_params);
3347}
3348
3349static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3350        .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3351        .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3352        .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3353};
3354
3355static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3356                                   struct mlxsw_sp_nexthop *nh)
3357{
3358        return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3359                                      &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3360}
3361
3362static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3363                                    struct mlxsw_sp_nexthop *nh)
3364{
3365        rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3366                               mlxsw_sp_nexthop_ht_params);
3367}
3368
3369static struct mlxsw_sp_nexthop *
3370mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3371                        struct mlxsw_sp_nexthop_key key)
3372{
3373        return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3374                                      mlxsw_sp_nexthop_ht_params);
3375}
3376
3377static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3378                                             enum mlxsw_sp_l3proto proto,
3379                                             u16 vr_id,
3380                                             u32 adj_index, u16 ecmp_size,
3381                                             u32 new_adj_index,
3382                                             u16 new_ecmp_size)
3383{
3384        char raleu_pl[MLXSW_REG_RALEU_LEN];
3385
3386        mlxsw_reg_raleu_pack(raleu_pl,
3387                             (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
3388                             adj_index, ecmp_size, new_adj_index,
3389                             new_ecmp_size);
3390        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3391}
3392
3393static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3394                                          struct mlxsw_sp_nexthop_group *nh_grp,
3395                                          u32 old_adj_index, u16 old_ecmp_size)
3396{
3397        struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3398        struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3399        int err;
3400
3401        list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3402                err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3403                                                        vr_entry->key.proto,
3404                                                        vr_entry->key.vr_id,
3405                                                        old_adj_index,
3406                                                        old_ecmp_size,
3407                                                        nhgi->adj_index,
3408                                                        nhgi->ecmp_size);
3409                if (err)
3410                        goto err_mass_update_vr;
3411        }
3412        return 0;
3413
3414err_mass_update_vr:
3415        list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3416                mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
3417                                                  vr_entry->key.vr_id,
3418                                                  nhgi->adj_index,
3419                                                  nhgi->ecmp_size,
3420                                                  old_adj_index, old_ecmp_size);
3421        return err;
3422}
3423
3424static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3425                                         u32 adj_index,
3426                                         struct mlxsw_sp_nexthop *nh,
3427                                         bool force, char *ratr_pl)
3428{
3429        struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3430        enum mlxsw_reg_ratr_op op;
3431        u16 rif_index;
3432
3433        rif_index = nh->rif ? nh->rif->rif_index :
3434                              mlxsw_sp->router->lb_rif_index;
3435        op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
3436                     MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
3437        mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET,
3438                            adj_index, rif_index);
3439        switch (nh->action) {
3440        case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3441                mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3442                break;
3443        case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3444                mlxsw_reg_ratr_trap_action_set(ratr_pl,
3445                                               MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3446                break;
3447        case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3448                mlxsw_reg_ratr_trap_action_set(ratr_pl,
3449                                               MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3450                mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
3451                break;
3452        default:
3453                WARN_ON_ONCE(1);
3454                return -EINVAL;
3455        }
3456        if (nh->counter_valid)
3457                mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3458        else
3459                mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3460
3461        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3462}
3463
3464int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3465                                struct mlxsw_sp_nexthop *nh, bool force,
3466                                char *ratr_pl)
3467{
3468        int i;
3469
3470        for (i = 0; i < nh->num_adj_entries; i++) {
3471                int err;
3472
3473                err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
3474                                                    nh, force, ratr_pl);
3475                if (err)
3476                        return err;
3477        }
3478
3479        return 0;
3480}
3481
3482static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3483                                          u32 adj_index,
3484                                          struct mlxsw_sp_nexthop *nh,
3485                                          bool force, char *ratr_pl)
3486{
3487        const struct mlxsw_sp_ipip_ops *ipip_ops;
3488
3489        ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3490        return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
3491                                        force, ratr_pl);
3492}
3493
3494static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3495                                        u32 adj_index,
3496                                        struct mlxsw_sp_nexthop *nh, bool force,
3497                                        char *ratr_pl)
3498{
3499        int i;
3500
3501        for (i = 0; i < nh->num_adj_entries; i++) {
3502                int err;
3503
3504                err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3505                                                     nh, force, ratr_pl);
3506                if (err)
3507                        return err;
3508        }
3509
3510        return 0;
3511}
3512
3513static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3514                                   struct mlxsw_sp_nexthop *nh, bool force,
3515                                   char *ratr_pl)
3516{
3517        /* When action is discard or trap, the nexthop must be
3518         * programmed as an Ethernet nexthop.
3519         */
3520        if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3521            nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3522            nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3523                return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
3524                                                   force, ratr_pl);
3525        else
3526                return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
3527                                                    force, ratr_pl);
3528}
3529
3530static int
3531mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3532                              struct mlxsw_sp_nexthop_group_info *nhgi,
3533                              bool reallocate)
3534{
3535        char ratr_pl[MLXSW_REG_RATR_LEN];
3536        u32 adj_index = nhgi->adj_index; /* base */
3537        struct mlxsw_sp_nexthop *nh;
3538        int i;
3539
3540        for (i = 0; i < nhgi->count; i++) {
3541                nh = &nhgi->nexthops[i];
3542
3543                if (!nh->should_offload) {
3544                        nh->offloaded = 0;
3545                        continue;
3546                }
3547
3548                if (nh->update || reallocate) {
3549                        int err = 0;
3550
3551                        err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
3552                                                      true, ratr_pl);
3553                        if (err)
3554                                return err;
3555                        nh->update = 0;
3556                        nh->offloaded = 1;
3557                }
3558                adj_index += nh->num_adj_entries;
3559        }
3560        return 0;
3561}
3562
3563static int
3564mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3565                                    struct mlxsw_sp_nexthop_group *nh_grp)
3566{
3567        struct mlxsw_sp_fib_entry *fib_entry;
3568        int err;
3569
3570        list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3571                err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3572                if (err)
3573                        return err;
3574        }
3575        return 0;
3576}
3577
3578struct mlxsw_sp_adj_grp_size_range {
3579        u16 start; /* Inclusive */
3580        u16 end; /* Inclusive */
3581};
3582
3583/* Ordered by range start value */
3584static const struct mlxsw_sp_adj_grp_size_range
3585mlxsw_sp1_adj_grp_size_ranges[] = {
3586        { .start = 1, .end = 64 },
3587        { .start = 512, .end = 512 },
3588        { .start = 1024, .end = 1024 },
3589        { .start = 2048, .end = 2048 },
3590        { .start = 4096, .end = 4096 },
3591};
3592
3593/* Ordered by range start value */
3594static const struct mlxsw_sp_adj_grp_size_range
3595mlxsw_sp2_adj_grp_size_ranges[] = {
3596        { .start = 1, .end = 128 },
3597        { .start = 256, .end = 256 },
3598        { .start = 512, .end = 512 },
3599        { .start = 1024, .end = 1024 },
3600        { .start = 2048, .end = 2048 },
3601        { .start = 4096, .end = 4096 },
3602};
3603
3604static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
3605                                           u16 *p_adj_grp_size)
3606{
3607        int i;
3608
3609        for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
3610                const struct mlxsw_sp_adj_grp_size_range *size_range;
3611
3612                size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3613
3614                if (*p_adj_grp_size >= size_range->start &&
3615                    *p_adj_grp_size <= size_range->end)
3616                        return;
3617
3618                if (*p_adj_grp_size <= size_range->end) {
3619                        *p_adj_grp_size = size_range->end;
3620                        return;
3621                }
3622        }
3623}
3624
3625static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
3626                                             u16 *p_adj_grp_size,
3627                                             unsigned int alloc_size)
3628{
3629        int i;
3630
3631        for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
3632                const struct mlxsw_sp_adj_grp_size_range *size_range;
3633
3634                size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3635
3636                if (alloc_size >= size_range->end) {
3637                        *p_adj_grp_size = size_range->end;
3638                        return;
3639                }
3640        }
3641}
3642
3643static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3644                                     u16 *p_adj_grp_size)
3645{
3646        unsigned int alloc_size;
3647        int err;
3648
3649        /* Round up the requested group size to the next size supported
3650         * by the device and make sure the request can be satisfied.
3651         */
3652        mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
3653        err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3654                                              MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3655                                              *p_adj_grp_size, &alloc_size);
3656        if (err)
3657                return err;
3658        /* It is possible the allocation results in more allocated
3659         * entries than requested. Try to use as much of them as
3660         * possible.
3661         */
3662        mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
3663
3664        return 0;
3665}
3666
3667static void
3668mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3669{
3670        int i, g = 0, sum_norm_weight = 0;
3671        struct mlxsw_sp_nexthop *nh;
3672
3673        for (i = 0; i < nhgi->count; i++) {
3674                nh = &nhgi->nexthops[i];
3675
3676                if (!nh->should_offload)
3677                        continue;
3678                if (g > 0)
3679                        g = gcd(nh->nh_weight, g);
3680                else
3681                        g = nh->nh_weight;
3682        }
3683
3684        for (i = 0; i < nhgi->count; i++) {
3685                nh = &nhgi->nexthops[i];
3686
3687                if (!nh->should_offload)
3688                        continue;
3689                nh->norm_nh_weight = nh->nh_weight / g;
3690                sum_norm_weight += nh->norm_nh_weight;
3691        }
3692
3693        nhgi->sum_norm_weight = sum_norm_weight;
3694}
3695
3696static void
3697mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3698{
3699        int i, weight = 0, lower_bound = 0;
3700        int total = nhgi->sum_norm_weight;
3701        u16 ecmp_size = nhgi->ecmp_size;
3702
3703        for (i = 0; i < nhgi->count; i++) {
3704                struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3705                int upper_bound;
3706
3707                if (!nh->should_offload)
3708                        continue;
3709                weight += nh->norm_nh_weight;
3710                upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3711                nh->num_adj_entries = upper_bound - lower_bound;
3712                lower_bound = upper_bound;
3713        }
3714}
3715
3716static struct mlxsw_sp_nexthop *
3717mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3718                     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3719
3720static void
3721mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3722                                        struct mlxsw_sp_nexthop_group *nh_grp)
3723{
3724        int i;
3725
3726        for (i = 0; i < nh_grp->nhgi->count; i++) {
3727                struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3728
3729                if (nh->offloaded)
3730                        nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3731                else
3732                        nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3733        }
3734}
3735
3736static void
3737__mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3738                                          struct mlxsw_sp_fib6_entry *fib6_entry)
3739{
3740        struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3741
3742        list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3743                struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3744                struct mlxsw_sp_nexthop *nh;
3745
3746                nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3747                if (nh && nh->offloaded)
3748                        fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3749                else
3750                        fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3751        }
3752}
3753
3754static void
3755mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3756                                        struct mlxsw_sp_nexthop_group *nh_grp)
3757{
3758        struct mlxsw_sp_fib6_entry *fib6_entry;
3759
3760        /* Unfortunately, in IPv6 the route and the nexthop are described by
3761         * the same struct, so we need to iterate over all the routes using the
3762         * nexthop group and set / clear the offload indication for them.
3763         */
3764        list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3765                            common.nexthop_group_node)
3766                __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3767}
3768
3769static void
3770mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3771                                        const struct mlxsw_sp_nexthop *nh,
3772                                        u16 bucket_index)
3773{
3774        struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
3775        bool offload = false, trap = false;
3776
3777        if (nh->offloaded) {
3778                if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3779                        trap = true;
3780                else
3781                        offload = true;
3782        }
3783        nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3784                                    bucket_index, offload, trap);
3785}
3786
3787static void
3788mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3789                                           struct mlxsw_sp_nexthop_group *nh_grp)
3790{
3791        int i;
3792
3793        /* Do not update the flags if the nexthop group is being destroyed
3794         * since:
3795         * 1. The nexthop objects is being deleted, in which case the flags are
3796         * irrelevant.
3797         * 2. The nexthop group was replaced by a newer group, in which case
3798         * the flags of the nexthop object were already updated based on the
3799         * new group.
3800         */
3801        if (nh_grp->can_destroy)
3802                return;
3803
3804        nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3805                             nh_grp->nhgi->adj_index_valid, false);
3806
3807        /* Update flags of individual nexthop buckets in case of a resilient
3808         * nexthop group.
3809         */
3810        if (!nh_grp->nhgi->is_resilient)
3811                return;
3812
3813        for (i = 0; i < nh_grp->nhgi->count; i++) {
3814                struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3815
3816                mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i);
3817        }
3818}
3819
3820static void
3821mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3822                                       struct mlxsw_sp_nexthop_group *nh_grp)
3823{
3824        switch (nh_grp->type) {
3825        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3826                mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3827                break;
3828        case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3829                mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3830                break;
3831        case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3832                mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
3833                break;
3834        }
3835}
3836
3837static int
3838mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3839                               struct mlxsw_sp_nexthop_group *nh_grp)
3840{
3841        struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3842        u16 ecmp_size, old_ecmp_size;
3843        struct mlxsw_sp_nexthop *nh;
3844        bool offload_change = false;
3845        u32 adj_index;
3846        bool old_adj_index_valid;
3847        u32 old_adj_index;
3848        int i, err2, err;
3849
3850        if (!nhgi->gateway)
3851                return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3852
3853        for (i = 0; i < nhgi->count; i++) {
3854                nh = &nhgi->nexthops[i];
3855
3856                if (nh->should_offload != nh->offloaded) {
3857                        offload_change = true;
3858                        if (nh->should_offload)
3859                                nh->update = 1;
3860                }
3861        }
3862        if (!offload_change) {
3863                /* Nothing was added or removed, so no need to reallocate. Just
3864                 * update MAC on existing adjacency indexes.
3865                 */
3866                err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3867                if (err) {
3868                        dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3869                        goto set_trap;
3870                }
3871                /* Flags of individual nexthop buckets might need to be
3872                 * updated.
3873                 */
3874                mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3875                return 0;
3876        }
3877        mlxsw_sp_nexthop_group_normalize(nhgi);
3878        if (!nhgi->sum_norm_weight) {
3879                /* No neigh of this group is connected so we just set
3880                 * the trap and let everthing flow through kernel.
3881                 */
3882                err = 0;
3883                goto set_trap;
3884        }
3885
3886        ecmp_size = nhgi->sum_norm_weight;
3887        err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3888        if (err)
3889                /* No valid allocation size available. */
3890                goto set_trap;
3891
3892        err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3893                                  ecmp_size, &adj_index);
3894        if (err) {
3895                /* We ran out of KVD linear space, just set the
3896                 * trap and let everything flow through kernel.
3897                 */
3898                dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3899                goto set_trap;
3900        }
3901        old_adj_index_valid = nhgi->adj_index_valid;
3902        old_adj_index = nhgi->adj_index;
3903        old_ecmp_size = nhgi->ecmp_size;
3904        nhgi->adj_index_valid = 1;
3905        nhgi->adj_index = adj_index;
3906        nhgi->ecmp_size = ecmp_size;
3907        mlxsw_sp_nexthop_group_rebalance(nhgi);
3908        err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3909        if (err) {
3910                dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3911                goto set_trap;
3912        }
3913
3914        mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3915
3916        if (!old_adj_index_valid) {
3917                /* The trap was set for fib entries, so we have to call
3918                 * fib entry update to unset it and use adjacency index.
3919                 */
3920                err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3921                if (err) {
3922                        dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3923                        goto set_trap;
3924                }
3925                return 0;
3926        }
3927
3928        err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3929                                             old_adj_index, old_ecmp_size);
3930        mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3931                           old_ecmp_size, old_adj_index);
3932        if (err) {
3933                dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3934                goto set_trap;
3935        }
3936
3937        return 0;
3938
3939set_trap:
3940        old_adj_index_valid = nhgi->adj_index_valid;
3941        nhgi->adj_index_valid = 0;
3942        for (i = 0; i < nhgi->count; i++) {
3943                nh = &nhgi->nexthops[i];
3944                nh->offloaded = 0;
3945        }
3946        err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3947        if (err2)
3948                dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3949        mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3950        if (old_adj_index_valid)
3951                mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3952                                   nhgi->ecmp_size, nhgi->adj_index);
3953        return err;
3954}
3955
3956static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3957                                            bool removing)
3958{
3959        if (!removing) {
3960                nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3961                nh->should_offload = 1;
3962        } else if (nh->nhgi->is_resilient) {
3963                nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
3964                nh->should_offload = 1;
3965        } else {
3966                nh->should_offload = 0;
3967        }
3968        nh->update = 1;
3969}
3970
3971static int
3972mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
3973                                    struct mlxsw_sp_neigh_entry *neigh_entry)
3974{
3975        struct neighbour *n, *old_n = neigh_entry->key.n;
3976        struct mlxsw_sp_nexthop *nh;
3977        bool entry_connected;
3978        u8 nud_state, dead;
3979        int err;
3980
3981        nh = list_first_entry(&neigh_entry->nexthop_list,
3982                              struct mlxsw_sp_nexthop, neigh_list_node);
3983
3984        n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3985        if (!n) {
3986                n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3987                if (IS_ERR(n))
3988                        return PTR_ERR(n);
3989                neigh_event_send(n, NULL);
3990        }
3991
3992        mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
3993        neigh_entry->key.n = n;
3994        err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
3995        if (err)
3996                goto err_neigh_entry_insert;
3997
3998        read_lock_bh(&n->lock);
3999        nud_state = n->nud_state;
4000        dead = n->dead;
4001        read_unlock_bh(&n->lock);
4002        entry_connected = nud_state & NUD_VALID && !dead;
4003
4004        list_for_each_entry(nh, &neigh_entry->nexthop_list,
4005                            neigh_list_node) {
4006                neigh_release(old_n);
4007                neigh_clone(n);
4008                __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
4009                mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4010        }
4011
4012        neigh_release(n);
4013
4014        return 0;
4015
4016err_neigh_entry_insert:
4017        neigh_entry->key.n = old_n;
4018        mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4019        neigh_release(n);
4020        return err;
4021}
4022
4023static void
4024mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
4025                              struct mlxsw_sp_neigh_entry *neigh_entry,
4026                              bool removing, bool dead)
4027{
4028        struct mlxsw_sp_nexthop *nh;
4029
4030        if (list_empty(&neigh_entry->nexthop_list))
4031                return;
4032
4033        if (dead) {
4034                int err;
4035
4036                err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
4037                                                          neigh_entry);
4038                if (err)
4039                        dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
4040                return;
4041        }
4042
4043        list_for_each_entry(nh, &neigh_entry->nexthop_list,
4044                            neigh_list_node) {
4045                __mlxsw_sp_nexthop_neigh_update(nh, removing);
4046                mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4047        }
4048}
4049
4050static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
4051                                      struct mlxsw_sp_rif *rif)
4052{
4053        if (nh->rif)
4054                return;
4055
4056        nh->rif = rif;
4057        list_add(&nh->rif_list_node, &rif->nexthop_list);
4058}
4059
4060static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
4061{
4062        if (!nh->rif)
4063                return;
4064
4065        list_del(&nh->rif_list_node);
4066        nh->rif = NULL;
4067}
4068
4069static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
4070                                       struct mlxsw_sp_nexthop *nh)
4071{
4072        struct mlxsw_sp_neigh_entry *neigh_entry;
4073        struct neighbour *n;
4074        u8 nud_state, dead;
4075        int err;
4076
4077        if (!nh->nhgi->gateway || nh->neigh_entry)
4078                return 0;
4079
4080        /* Take a reference of neigh here ensuring that neigh would
4081         * not be destructed before the nexthop entry is finished.
4082         * The reference is taken either in neigh_lookup() or
4083         * in neigh_create() in case n is not found.
4084         */
4085        n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4086        if (!n) {
4087                n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4088                if (IS_ERR(n))
4089                        return PTR_ERR(n);
4090                neigh_event_send(n, NULL);
4091        }
4092        neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
4093        if (!neigh_entry) {
4094                neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
4095                if (IS_ERR(neigh_entry)) {
4096                        err = -EINVAL;
4097                        goto err_neigh_entry_create;
4098                }
4099        }
4100
4101        /* If that is the first nexthop connected to that neigh, add to
4102         * nexthop_neighs_list
4103         */
4104        if (list_empty(&neigh_entry->nexthop_list))
4105                list_add_tail(&neigh_entry->nexthop_neighs_list_node,
4106                              &mlxsw_sp->router->nexthop_neighs_list);
4107
4108        nh->neigh_entry = neigh_entry;
4109        list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
4110        read_lock_bh(&n->lock);
4111        nud_state = n->nud_state;
4112        dead = n->dead;
4113        read_unlock_bh(&n->lock);
4114        __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
4115
4116        return 0;
4117
4118err_neigh_entry_create:
4119        neigh_release(n);
4120        return err;
4121}
4122
4123static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4124                                        struct mlxsw_sp_nexthop *nh)
4125{
4126        struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4127        struct neighbour *n;
4128
4129        if (!neigh_entry)
4130                return;
4131        n = neigh_entry->key.n;
4132
4133        __mlxsw_sp_nexthop_neigh_update(nh, true);
4134        list_del(&nh->neigh_list_node);
4135        nh->neigh_entry = NULL;
4136
4137        /* If that is the last nexthop connected to that neigh, remove from
4138         * nexthop_neighs_list
4139         */
4140        if (list_empty(&neigh_entry->nexthop_list))
4141                list_del(&neigh_entry->nexthop_neighs_list_node);
4142
4143        if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
4144                mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4145
4146        neigh_release(n);
4147}
4148
4149static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4150{
4151        struct net_device *ul_dev;
4152        bool is_up;
4153
4154        rcu_read_lock();
4155        ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4156        is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4157        rcu_read_unlock();
4158
4159        return is_up;
4160}
4161
4162static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4163                                       struct mlxsw_sp_nexthop *nh,
4164                                       struct mlxsw_sp_ipip_entry *ipip_entry)
4165{
4166        bool removing;
4167
4168        if (!nh->nhgi->gateway || nh->ipip_entry)
4169                return;
4170
4171        nh->ipip_entry = ipip_entry;
4172        removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
4173        __mlxsw_sp_nexthop_neigh_update(nh, removing);
4174        mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
4175}
4176
4177static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4178                                       struct mlxsw_sp_nexthop *nh)
4179{
4180        struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4181
4182        if (!ipip_entry)
4183                return;
4184
4185        __mlxsw_sp_nexthop_neigh_update(nh, true);
4186        nh->ipip_entry = NULL;
4187}
4188
4189static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4190                                        const struct fib_nh *fib_nh,
4191                                        enum mlxsw_sp_ipip_type *p_ipipt)
4192{
4193        struct net_device *dev = fib_nh->fib_nh_dev;
4194
4195        return dev &&
4196               fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4197               mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
4198}
4199
4200static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4201                                      struct mlxsw_sp_nexthop *nh,
4202                                      const struct net_device *dev)
4203{
4204        const struct mlxsw_sp_ipip_ops *ipip_ops;
4205        struct mlxsw_sp_ipip_entry *ipip_entry;
4206        struct mlxsw_sp_rif *rif;
4207        int err;
4208
4209        ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4210        if (ipip_entry) {
4211                ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4212                if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4213                        nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4214                        mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4215                        return 0;
4216                }
4217        }
4218
4219        nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4220        rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4221        if (!rif)
4222                return 0;
4223
4224        mlxsw_sp_nexthop_rif_init(nh, rif);
4225        err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4226        if (err)
4227                goto err_neigh_init;
4228
4229        return 0;
4230
4231err_neigh_init:
4232        mlxsw_sp_nexthop_rif_fini(nh);
4233        return err;
4234}
4235
4236static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4237                                       struct mlxsw_sp_nexthop *nh)
4238{
4239        switch (nh->type) {
4240        case MLXSW_SP_NEXTHOP_TYPE_ETH:
4241                mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4242                mlxsw_sp_nexthop_rif_fini(nh);
4243                break;
4244        case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4245                mlxsw_sp_nexthop_rif_fini(nh);
4246                mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4247                break;
4248        }
4249}
4250
4251static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4252                                  struct mlxsw_sp_nexthop_group *nh_grp,
4253                                  struct mlxsw_sp_nexthop *nh,
4254                                  struct fib_nh *fib_nh)
4255{
4256        struct net_device *dev = fib_nh->fib_nh_dev;
4257        struct in_device *in_dev;
4258        int err;
4259
4260        nh->nhgi = nh_grp->nhgi;
4261        nh->key.fib_nh = fib_nh;
4262#ifdef CONFIG_IP_ROUTE_MULTIPATH
4263        nh->nh_weight = fib_nh->fib_nh_weight;
4264#else
4265        nh->nh_weight = 1;
4266#endif
4267        memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4268        nh->neigh_tbl = &arp_tbl;
4269        err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4270        if (err)
4271                return err;
4272
4273        mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4274        list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4275
4276        if (!dev)
4277                return 0;
4278        nh->ifindex = dev->ifindex;
4279
4280        rcu_read_lock();
4281        in_dev = __in_dev_get_rcu(dev);
4282        if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4283            fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4284                rcu_read_unlock();
4285                return 0;
4286        }
4287        rcu_read_unlock();
4288
4289        err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4290        if (err)
4291                goto err_nexthop_neigh_init;
4292
4293        return 0;
4294
4295err_nexthop_neigh_init:
4296        mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4297        return err;
4298}
4299
4300static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4301                                   struct mlxsw_sp_nexthop *nh)
4302{
4303        mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4304        list_del(&nh->router_list_node);
4305        mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4306        mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4307}
4308
4309static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4310                                    unsigned long event, struct fib_nh *fib_nh)
4311{
4312        struct mlxsw_sp_nexthop_key key;
4313        struct mlxsw_sp_nexthop *nh;
4314
4315        key.fib_nh = fib_nh;
4316        nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4317        if (!nh)
4318                return;
4319
4320        switch (event) {
4321        case FIB_EVENT_NH_ADD:
4322                mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4323                break;
4324        case FIB_EVENT_NH_DEL:
4325                mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4326                break;
4327        }
4328
4329        mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4330}
4331
4332static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4333                                        struct mlxsw_sp_rif *rif)
4334{
4335        struct mlxsw_sp_nexthop *nh;
4336        bool removing;
4337
4338        list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4339                switch (nh->type) {
4340                case MLXSW_SP_NEXTHOP_TYPE_ETH:
4341                        removing = false;
4342                        break;
4343                case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4344                        removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4345                        break;
4346                default:
4347                        WARN_ON(1);
4348                        continue;
4349                }
4350
4351                __mlxsw_sp_nexthop_neigh_update(nh, removing);
4352                mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4353        }
4354}
4355
4356static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4357                                         struct mlxsw_sp_rif *old_rif,
4358                                         struct mlxsw_sp_rif *new_rif)
4359{
4360        struct mlxsw_sp_nexthop *nh;
4361
4362        list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4363        list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4364                nh->rif = new_rif;
4365        mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4366}
4367
4368static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4369                                           struct mlxsw_sp_rif *rif)
4370{
4371        struct mlxsw_sp_nexthop *nh, *tmp;
4372
4373        list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4374                mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4375                mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4376        }
4377}
4378
4379static void
4380mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
4381                             const struct mlxsw_sp_nexthop_group *nh_grp,
4382                             unsigned long *activity)
4383{
4384        char *ratrad_pl;
4385        int i, err;
4386
4387        ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
4388        if (!ratrad_pl)
4389                return;
4390
4391        mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index,
4392                              nh_grp->nhgi->count);
4393        err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl);
4394        if (err)
4395                goto out;
4396
4397        for (i = 0; i < nh_grp->nhgi->count; i++) {
4398                if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i))
4399                        continue;
4400                bitmap_set(activity, i, 1);
4401        }
4402
4403out:
4404        kfree(ratrad_pl);
4405}
4406
4407#define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
4408
4409static void
4410mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
4411                                const struct mlxsw_sp_nexthop_group *nh_grp)
4412{
4413        unsigned long *activity;
4414
4415        activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL);
4416        if (!activity)
4417                return;
4418
4419        mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
4420        nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4421                                        nh_grp->nhgi->count, activity);
4422
4423        bitmap_free(activity);
4424}
4425
4426static void
4427mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
4428{
4429        unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
4430
4431        mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw,
4432                               msecs_to_jiffies(interval));
4433}
4434
4435static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
4436{
4437        struct mlxsw_sp_nexthop_group_info *nhgi;
4438        struct mlxsw_sp_router *router;
4439        bool reschedule = false;
4440
4441        router = container_of(work, struct mlxsw_sp_router,
4442                              nh_grp_activity_dw.work);
4443
4444        mutex_lock(&router->lock);
4445
4446        list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
4447                mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp);
4448                reschedule = true;
4449        }
4450
4451        mutex_unlock(&router->lock);
4452
4453        if (!reschedule)
4454                return;
4455        mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp);
4456}
4457
4458static int
4459mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4460                                     const struct nh_notifier_single_info *nh,
4461                                     struct netlink_ext_ack *extack)
4462{
4463        int err = -EINVAL;
4464
4465        if (nh->is_fdb)
4466                NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4467        else if (nh->has_encap)
4468                NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4469        else
4470                err = 0;
4471
4472        return err;
4473}
4474
4475static int
4476mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4477                                          const struct nh_notifier_single_info *nh,
4478                                          struct netlink_ext_ack *extack)
4479{
4480        int err;
4481
4482        err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4483        if (err)
4484                return err;
4485
4486        /* Device only nexthops with an IPIP device are programmed as
4487         * encapsulating adjacency entries.
4488         */
4489        if (!nh->gw_family && !nh->is_reject &&
4490            !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
4491                NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4492                return -EINVAL;
4493        }
4494
4495        return 0;
4496}
4497
4498static int
4499mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4500                                    const struct nh_notifier_grp_info *nh_grp,
4501                                    struct netlink_ext_ack *extack)
4502{
4503        int i;
4504
4505        if (nh_grp->is_fdb) {
4506                NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4507                return -EINVAL;
4508        }
4509
4510        for (i = 0; i < nh_grp->num_nh; i++) {
4511                const struct nh_notifier_single_info *nh;
4512                int err;
4513
4514                nh = &nh_grp->nh_entries[i].nh;
4515                err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4516                                                                extack);
4517                if (err)
4518                        return err;
4519        }
4520
4521        return 0;
4522}
4523
4524static int
4525mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
4526                                             const struct nh_notifier_res_table_info *nh_res_table,
4527                                             struct netlink_ext_ack *extack)
4528{
4529        unsigned int alloc_size;
4530        bool valid_size = false;
4531        int err, i;
4532
4533        if (nh_res_table->num_nh_buckets < 32) {
4534                NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
4535                return -EINVAL;
4536        }
4537
4538        for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
4539                const struct mlxsw_sp_adj_grp_size_range *size_range;
4540
4541                size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
4542
4543                if (nh_res_table->num_nh_buckets >= size_range->start &&
4544                    nh_res_table->num_nh_buckets <= size_range->end) {
4545                        valid_size = true;
4546                        break;
4547                }
4548        }
4549
4550        if (!valid_size) {
4551                NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
4552                return -EINVAL;
4553        }
4554
4555        err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
4556                                              MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4557                                              nh_res_table->num_nh_buckets,
4558                                              &alloc_size);
4559        if (err || nh_res_table->num_nh_buckets != alloc_size) {
4560                NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
4561                return -EINVAL;
4562        }
4563
4564        return 0;
4565}
4566
4567static int
4568mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
4569                                        const struct nh_notifier_res_table_info *nh_res_table,
4570                                        struct netlink_ext_ack *extack)
4571{
4572        int err;
4573        u16 i;
4574
4575        err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
4576                                                           nh_res_table,
4577                                                           extack);
4578        if (err)
4579                return err;
4580
4581        for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
4582                const struct nh_notifier_single_info *nh;
4583                int err;
4584
4585                nh = &nh_res_table->nhs[i];
4586                err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4587                                                                extack);
4588                if (err)
4589                        return err;
4590        }
4591
4592        return 0;
4593}
4594
4595static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4596                                         unsigned long event,
4597                                         struct nh_notifier_info *info)
4598{
4599        struct nh_notifier_single_info *nh;
4600
4601        if (event != NEXTHOP_EVENT_REPLACE &&
4602            event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
4603            event != NEXTHOP_EVENT_BUCKET_REPLACE)
4604                return 0;
4605
4606        switch (info->type) {
4607        case NH_NOTIFIER_INFO_TYPE_SINGLE:
4608                return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
4609                                                            info->extack);
4610        case NH_NOTIFIER_INFO_TYPE_GRP:
4611                return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4612                                                           info->nh_grp,
4613                                                           info->extack);
4614        case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4615                return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
4616                                                               info->nh_res_table,
4617                                                               info->extack);
4618        case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
4619                nh = &info->nh_res_bucket->new_nh;
4620                return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4621                                                                 info->extack);
4622        default:
4623                NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4624                return -EOPNOTSUPP;
4625        }
4626}
4627
4628static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4629                                            const struct nh_notifier_info *info)
4630{
4631        const struct net_device *dev;
4632
4633        switch (info->type) {
4634        case NH_NOTIFIER_INFO_TYPE_SINGLE:
4635                dev = info->nh->dev;
4636                return info->nh->gw_family || info->