linux/drivers/infiniband/ulp/ipoib/ipoib_main.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include "ipoib.h"
  36
  37#include <linux/module.h>
  38
  39#include <linux/init.h>
  40#include <linux/slab.h>
  41#include <linux/kernel.h>
  42#include <linux/vmalloc.h>
  43
  44#include <linux/if_arp.h>       /* For ARPHRD_xxx */
  45
  46#include <linux/ip.h>
  47#include <linux/in.h>
  48
  49#include <linux/jhash.h>
  50#include <net/arp.h>
  51#include <net/addrconf.h>
  52#include <linux/inetdevice.h>
  53#include <rdma/ib_cache.h>
  54
  55#define DRV_VERSION "1.0.0"
  56
  57const char ipoib_driver_version[] = DRV_VERSION;
  58
  59MODULE_AUTHOR("Roland Dreier");
  60MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
  61MODULE_LICENSE("Dual BSD/GPL");
  62
  63int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
  64int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;
  65
  66module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
  67MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
  68module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
  69MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");
  70
  71#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
  72int ipoib_debug_level;
  73
  74module_param_named(debug_level, ipoib_debug_level, int, 0644);
  75MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
  76#endif
  77
  78struct ipoib_path_iter {
  79        struct net_device *dev;
  80        struct ipoib_path  path;
  81};
  82
  83static const u8 ipv4_bcast_addr[] = {
  84        0x00, 0xff, 0xff, 0xff,
  85        0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
  86        0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
  87};
  88
  89struct workqueue_struct *ipoib_workqueue;
  90
  91struct ib_sa_client ipoib_sa_client;
  92
  93static void ipoib_add_one(struct ib_device *device);
  94static void ipoib_remove_one(struct ib_device *device, void *client_data);
  95static void ipoib_neigh_reclaim(struct rcu_head *rp);
  96static struct net_device *ipoib_get_net_dev_by_params(
  97                struct ib_device *dev, u8 port, u16 pkey,
  98                const union ib_gid *gid, const struct sockaddr *addr,
  99                void *client_data);
 100static int ipoib_set_mac(struct net_device *dev, void *addr);
 101static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
 102                       int cmd);
 103
 104static struct ib_client ipoib_client = {
 105        .name   = "ipoib",
 106        .add    = ipoib_add_one,
 107        .remove = ipoib_remove_one,
 108        .get_net_dev_by_params = ipoib_get_net_dev_by_params,
 109};
 110
 111#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 112static int ipoib_netdev_event(struct notifier_block *this,
 113                              unsigned long event, void *ptr)
 114{
 115        struct netdev_notifier_info *ni = ptr;
 116        struct net_device *dev = ni->dev;
 117
 118        if (dev->netdev_ops->ndo_open != ipoib_open)
 119                return NOTIFY_DONE;
 120
 121        switch (event) {
 122        case NETDEV_REGISTER:
 123                ipoib_create_debug_files(dev);
 124                break;
 125        case NETDEV_CHANGENAME:
 126                ipoib_delete_debug_files(dev);
 127                ipoib_create_debug_files(dev);
 128                break;
 129        case NETDEV_UNREGISTER:
 130                ipoib_delete_debug_files(dev);
 131                break;
 132        }
 133
 134        return NOTIFY_DONE;
 135}
 136#endif
 137
 138int ipoib_open(struct net_device *dev)
 139{
 140        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 141
 142        ipoib_dbg(priv, "bringing up interface\n");
 143
 144        netif_carrier_off(dev);
 145
 146        set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 147
 148        priv->sm_fullmember_sendonly_support = false;
 149
 150        if (ipoib_ib_dev_open(dev)) {
 151                if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
 152                        return 0;
 153                goto err_disable;
 154        }
 155
 156        ipoib_ib_dev_up(dev);
 157
 158        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
 159                struct ipoib_dev_priv *cpriv;
 160
 161                /* Bring up any child interfaces too */
 162                down_read(&priv->vlan_rwsem);
 163                list_for_each_entry(cpriv, &priv->child_intfs, list) {
 164                        int flags;
 165
 166                        flags = cpriv->dev->flags;
 167                        if (flags & IFF_UP)
 168                                continue;
 169
 170                        dev_change_flags(cpriv->dev, flags | IFF_UP, NULL);
 171                }
 172                up_read(&priv->vlan_rwsem);
 173        }
 174
 175        netif_start_queue(dev);
 176
 177        return 0;
 178
 179err_disable:
 180        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 181
 182        return -EINVAL;
 183}
 184
 185static int ipoib_stop(struct net_device *dev)
 186{
 187        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 188
 189        ipoib_dbg(priv, "stopping interface\n");
 190
 191        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 192
 193        netif_stop_queue(dev);
 194
 195        ipoib_ib_dev_down(dev);
 196        ipoib_ib_dev_stop(dev);
 197
 198        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
 199                struct ipoib_dev_priv *cpriv;
 200
 201                /* Bring down any child interfaces too */
 202                down_read(&priv->vlan_rwsem);
 203                list_for_each_entry(cpriv, &priv->child_intfs, list) {
 204                        int flags;
 205
 206                        flags = cpriv->dev->flags;
 207                        if (!(flags & IFF_UP))
 208                                continue;
 209
 210                        dev_change_flags(cpriv->dev, flags & ~IFF_UP, NULL);
 211                }
 212                up_read(&priv->vlan_rwsem);
 213        }
 214
 215        return 0;
 216}
 217
 218static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
 219{
 220        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 221
 222        if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
 223                features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
 224
 225        return features;
 226}
 227
 228static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
 229{
 230        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 231        int ret = 0;
 232
 233        /* dev->mtu > 2K ==> connected mode */
 234        if (ipoib_cm_admin_enabled(dev)) {
 235                if (new_mtu > ipoib_cm_max_mtu(dev))
 236                        return -EINVAL;
 237
 238                if (new_mtu > priv->mcast_mtu)
 239                        ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
 240                                   priv->mcast_mtu);
 241
 242                dev->mtu = new_mtu;
 243                return 0;
 244        }
 245
 246        if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
 247            new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
 248                return -EINVAL;
 249
 250        priv->admin_mtu = new_mtu;
 251
 252        if (priv->mcast_mtu < priv->admin_mtu)
 253                ipoib_dbg(priv, "MTU must be smaller than the underlying "
 254                                "link layer MTU - 4 (%u)\n", priv->mcast_mtu);
 255
 256        new_mtu = min(priv->mcast_mtu, priv->admin_mtu);
 257
 258        if (priv->rn_ops->ndo_change_mtu) {
 259                bool carrier_status = netif_carrier_ok(dev);
 260
 261                netif_carrier_off(dev);
 262
 263                /* notify lower level on the real mtu */
 264                ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);
 265
 266                if (carrier_status)
 267                        netif_carrier_on(dev);
 268        } else {
 269                dev->mtu = new_mtu;
 270        }
 271
 272        return ret;
 273}
 274
 275static void ipoib_get_stats(struct net_device *dev,
 276                            struct rtnl_link_stats64 *stats)
 277{
 278        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 279
 280        if (priv->rn_ops->ndo_get_stats64)
 281                priv->rn_ops->ndo_get_stats64(dev, stats);
 282        else
 283                netdev_stats_to_stats64(stats, &dev->stats);
 284}
 285
 286/* Called with an RCU read lock taken */
 287static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
 288                                        struct net_device *dev)
 289{
 290        struct net *net = dev_net(dev);
 291        struct in_device *in_dev;
 292        struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
 293        struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
 294        __be32 ret_addr;
 295
 296        switch (addr->sa_family) {
 297        case AF_INET:
 298                in_dev = in_dev_get(dev);
 299                if (!in_dev)
 300                        return false;
 301
 302                ret_addr = inet_confirm_addr(net, in_dev, 0,
 303                                             addr_in->sin_addr.s_addr,
 304                                             RT_SCOPE_HOST);
 305                in_dev_put(in_dev);
 306                if (ret_addr)
 307                        return true;
 308
 309                break;
 310        case AF_INET6:
 311                if (IS_ENABLED(CONFIG_IPV6) &&
 312                    ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))
 313                        return true;
 314
 315                break;
 316        }
 317        return false;
 318}
 319
 320/**
 321 * Find the master net_device on top of the given net_device.
 322 * @dev: base IPoIB net_device
 323 *
 324 * Returns the master net_device with a reference held, or the same net_device
 325 * if no master exists.
 326 */
 327static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
 328{
 329        struct net_device *master;
 330
 331        rcu_read_lock();
 332        master = netdev_master_upper_dev_get_rcu(dev);
 333        if (master)
 334                dev_hold(master);
 335        rcu_read_unlock();
 336
 337        if (master)
 338                return master;
 339
 340        dev_hold(dev);
 341        return dev;
 342}
 343
 344struct ipoib_walk_data {
 345        const struct sockaddr *addr;
 346        struct net_device *result;
 347};
 348
 349static int ipoib_upper_walk(struct net_device *upper, void *_data)
 350{
 351        struct ipoib_walk_data *data = _data;
 352        int ret = 0;
 353
 354        if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
 355                dev_hold(upper);
 356                data->result = upper;
 357                ret = 1;
 358        }
 359
 360        return ret;
 361}
 362
 363/**
 364 * Find a net_device matching the given address, which is an upper device of
 365 * the given net_device.
 366 * @addr: IP address to look for.
 367 * @dev: base IPoIB net_device
 368 *
 369 * If found, returns the net_device with a reference held. Otherwise return
 370 * NULL.
 371 */
 372static struct net_device *ipoib_get_net_dev_match_addr(
 373                const struct sockaddr *addr, struct net_device *dev)
 374{
 375        struct ipoib_walk_data data = {
 376                .addr = addr,
 377        };
 378
 379        rcu_read_lock();
 380        if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
 381                dev_hold(dev);
 382                data.result = dev;
 383                goto out;
 384        }
 385
 386        netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &data);
 387out:
 388        rcu_read_unlock();
 389        return data.result;
 390}
 391
 392/* returns the number of IPoIB netdevs on top a given ipoib device matching a
 393 * pkey_index and address, if one exists.
 394 *
 395 * @found_net_dev: contains a matching net_device if the return value >= 1,
 396 * with a reference held. */
 397static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
 398                                     const union ib_gid *gid,
 399                                     u16 pkey_index,
 400                                     const struct sockaddr *addr,
 401                                     int nesting,
 402                                     struct net_device **found_net_dev)
 403{
 404        struct ipoib_dev_priv *child_priv;
 405        struct net_device *net_dev = NULL;
 406        int matches = 0;
 407
 408        if (priv->pkey_index == pkey_index &&
 409            (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {
 410                if (!addr) {
 411                        net_dev = ipoib_get_master_net_dev(priv->dev);
 412                } else {
 413                        /* Verify the net_device matches the IP address, as
 414                         * IPoIB child devices currently share a GID. */
 415                        net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);
 416                }
 417                if (net_dev) {
 418                        if (!*found_net_dev)
 419                                *found_net_dev = net_dev;
 420                        else
 421                                dev_put(net_dev);
 422                        ++matches;
 423                }
 424        }
 425
 426        /* Check child interfaces */
 427        down_read_nested(&priv->vlan_rwsem, nesting);
 428        list_for_each_entry(child_priv, &priv->child_intfs, list) {
 429                matches += ipoib_match_gid_pkey_addr(child_priv, gid,
 430                                                    pkey_index, addr,
 431                                                    nesting + 1,
 432                                                    found_net_dev);
 433                if (matches > 1)
 434                        break;
 435        }
 436        up_read(&priv->vlan_rwsem);
 437
 438        return matches;
 439}
 440
 441/* Returns the number of matching net_devs found (between 0 and 2). Also
 442 * return the matching net_device in the @net_dev parameter, holding a
 443 * reference to the net_device, if the number of matches >= 1 */
 444static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u8 port,
 445                                         u16 pkey_index,
 446                                         const union ib_gid *gid,
 447                                         const struct sockaddr *addr,
 448                                         struct net_device **net_dev)
 449{
 450        struct ipoib_dev_priv *priv;
 451        int matches = 0;
 452
 453        *net_dev = NULL;
 454
 455        list_for_each_entry(priv, dev_list, list) {
 456                if (priv->port != port)
 457                        continue;
 458
 459                matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
 460                                                     addr, 0, net_dev);
 461                if (matches > 1)
 462                        break;
 463        }
 464
 465        return matches;
 466}
 467
 468static struct net_device *ipoib_get_net_dev_by_params(
 469                struct ib_device *dev, u8 port, u16 pkey,
 470                const union ib_gid *gid, const struct sockaddr *addr,
 471                void *client_data)
 472{
 473        struct net_device *net_dev;
 474        struct list_head *dev_list = client_data;
 475        u16 pkey_index;
 476        int matches;
 477        int ret;
 478
 479        if (!rdma_protocol_ib(dev, port))
 480                return NULL;
 481
 482        ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);
 483        if (ret)
 484                return NULL;
 485
 486        if (!dev_list)
 487                return NULL;
 488
 489        /* See if we can find a unique device matching the L2 parameters */
 490        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
 491                                                gid, NULL, &net_dev);
 492
 493        switch (matches) {
 494        case 0:
 495                return NULL;
 496        case 1:
 497                return net_dev;
 498        }
 499
 500        dev_put(net_dev);
 501
 502        /* Couldn't find a unique device with L2 parameters only. Use L3
 503         * address to uniquely match the net device */
 504        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
 505                                                gid, addr, &net_dev);
 506        switch (matches) {
 507        case 0:
 508                return NULL;
 509        default:
 510                dev_warn_ratelimited(&dev->dev,
 511                                     "duplicate IP address detected\n");
 512                /* Fall through */
 513        case 1:
 514                return net_dev;
 515        }
 516}
 517
 518int ipoib_set_mode(struct net_device *dev, const char *buf)
 519{
 520        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 521
 522        if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
 523             !strcmp(buf, "connected\n")) ||
 524             (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
 525             !strcmp(buf, "datagram\n"))) {
 526                return 0;
 527        }
 528
 529        /* flush paths if we switch modes so that connections are restarted */
 530        if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
 531                set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
 532                ipoib_warn(priv, "enabling connected mode "
 533                           "will cause multicast packet drops\n");
 534                netdev_update_features(dev);
 535                dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
 536                rtnl_unlock();
 537                priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
 538
 539                ipoib_flush_paths(dev);
 540                return (!rtnl_trylock()) ? -EBUSY : 0;
 541        }
 542
 543        if (!strcmp(buf, "datagram\n")) {
 544                clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
 545                netdev_update_features(dev);
 546                dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
 547                rtnl_unlock();
 548                ipoib_flush_paths(dev);
 549                return (!rtnl_trylock()) ? -EBUSY : 0;
 550        }
 551
 552        return -EINVAL;
 553}
 554
 555struct ipoib_path *__path_find(struct net_device *dev, void *gid)
 556{
 557        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 558        struct rb_node *n = priv->path_tree.rb_node;
 559        struct ipoib_path *path;
 560        int ret;
 561
 562        while (n) {
 563                path = rb_entry(n, struct ipoib_path, rb_node);
 564
 565                ret = memcmp(gid, path->pathrec.dgid.raw,
 566                             sizeof (union ib_gid));
 567
 568                if (ret < 0)
 569                        n = n->rb_left;
 570                else if (ret > 0)
 571                        n = n->rb_right;
 572                else
 573                        return path;
 574        }
 575
 576        return NULL;
 577}
 578
 579static int __path_add(struct net_device *dev, struct ipoib_path *path)
 580{
 581        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 582        struct rb_node **n = &priv->path_tree.rb_node;
 583        struct rb_node *pn = NULL;
 584        struct ipoib_path *tpath;
 585        int ret;
 586
 587        while (*n) {
 588                pn = *n;
 589                tpath = rb_entry(pn, struct ipoib_path, rb_node);
 590
 591                ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,
 592                             sizeof (union ib_gid));
 593                if (ret < 0)
 594                        n = &pn->rb_left;
 595                else if (ret > 0)
 596                        n = &pn->rb_right;
 597                else
 598                        return -EEXIST;
 599        }
 600
 601        rb_link_node(&path->rb_node, pn, n);
 602        rb_insert_color(&path->rb_node, &priv->path_tree);
 603
 604        list_add_tail(&path->list, &priv->path_list);
 605
 606        return 0;
 607}
 608
 609static void path_free(struct net_device *dev, struct ipoib_path *path)
 610{
 611        struct sk_buff *skb;
 612
 613        while ((skb = __skb_dequeue(&path->queue)))
 614                dev_kfree_skb_irq(skb);
 615
 616        ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);
 617
 618        /* remove all neigh connected to this path */
 619        ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
 620
 621        if (path->ah)
 622                ipoib_put_ah(path->ah);
 623
 624        kfree(path);
 625}
 626
 627#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 628
 629struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
 630{
 631        struct ipoib_path_iter *iter;
 632
 633        iter = kmalloc(sizeof(*iter), GFP_KERNEL);
 634        if (!iter)
 635                return NULL;
 636
 637        iter->dev = dev;
 638        memset(iter->path.pathrec.dgid.raw, 0, 16);
 639
 640        if (ipoib_path_iter_next(iter)) {
 641                kfree(iter);
 642                return NULL;
 643        }
 644
 645        return iter;
 646}
 647
 648int ipoib_path_iter_next(struct ipoib_path_iter *iter)
 649{
 650        struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);
 651        struct rb_node *n;
 652        struct ipoib_path *path;
 653        int ret = 1;
 654
 655        spin_lock_irq(&priv->lock);
 656
 657        n = rb_first(&priv->path_tree);
 658
 659        while (n) {
 660                path = rb_entry(n, struct ipoib_path, rb_node);
 661
 662                if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,
 663                           sizeof (union ib_gid)) < 0) {
 664                        iter->path = *path;
 665                        ret = 0;
 666                        break;
 667                }
 668
 669                n = rb_next(n);
 670        }
 671
 672        spin_unlock_irq(&priv->lock);
 673
 674        return ret;
 675}
 676
 677void ipoib_path_iter_read(struct ipoib_path_iter *iter,
 678                          struct ipoib_path *path)
 679{
 680        *path = iter->path;
 681}
 682
 683#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */
 684
 685void ipoib_mark_paths_invalid(struct net_device *dev)
 686{
 687        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 688        struct ipoib_path *path, *tp;
 689
 690        spin_lock_irq(&priv->lock);
 691
 692        list_for_each_entry_safe(path, tp, &priv->path_list, list) {
 693                ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
 694                          be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
 695                          path->pathrec.dgid.raw);
 696                if (path->ah)
 697                        path->ah->valid = 0;
 698        }
 699
 700        spin_unlock_irq(&priv->lock);
 701}
 702
 703static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
 704{
 705        struct ipoib_pseudo_header *phdr;
 706
 707        phdr = skb_push(skb, sizeof(*phdr));
 708        memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
 709}
 710
 711void ipoib_flush_paths(struct net_device *dev)
 712{
 713        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 714        struct ipoib_path *path, *tp;
 715        LIST_HEAD(remove_list);
 716        unsigned long flags;
 717
 718        netif_tx_lock_bh(dev);
 719        spin_lock_irqsave(&priv->lock, flags);
 720
 721        list_splice_init(&priv->path_list, &remove_list);
 722
 723        list_for_each_entry(path, &remove_list, list)
 724                rb_erase(&path->rb_node, &priv->path_tree);
 725
 726        list_for_each_entry_safe(path, tp, &remove_list, list) {
 727                if (path->query)
 728                        ib_sa_cancel_query(path->query_id, path->query);
 729                spin_unlock_irqrestore(&priv->lock, flags);
 730                netif_tx_unlock_bh(dev);
 731                wait_for_completion(&path->done);
 732                path_free(dev, path);
 733                netif_tx_lock_bh(dev);
 734                spin_lock_irqsave(&priv->lock, flags);
 735        }
 736
 737        spin_unlock_irqrestore(&priv->lock, flags);
 738        netif_tx_unlock_bh(dev);
 739}
 740
 741static void path_rec_completion(int status,
 742                                struct sa_path_rec *pathrec,
 743                                void *path_ptr)
 744{
 745        struct ipoib_path *path = path_ptr;
 746        struct net_device *dev = path->dev;
 747        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 748        struct ipoib_ah *ah = NULL;
 749        struct ipoib_ah *old_ah = NULL;
 750        struct ipoib_neigh *neigh, *tn;
 751        struct sk_buff_head skqueue;
 752        struct sk_buff *skb;
 753        unsigned long flags;
 754
 755        if (!status)
 756                ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
 757                          be32_to_cpu(sa_path_get_dlid(pathrec)),
 758                          pathrec->dgid.raw);
 759        else
 760                ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
 761                          status, path->pathrec.dgid.raw);
 762
 763        skb_queue_head_init(&skqueue);
 764
 765        if (!status) {
 766                struct rdma_ah_attr av;
 767
 768                if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
 769                                               pathrec, &av, NULL)) {
 770                        ah = ipoib_create_ah(dev, priv->pd, &av);
 771                        rdma_destroy_ah_attr(&av);
 772                }
 773        }
 774
 775        spin_lock_irqsave(&priv->lock, flags);
 776
 777        if (!IS_ERR_OR_NULL(ah)) {
 778                /*
 779                 * pathrec.dgid is used as the database key from the LLADDR,
 780                 * it must remain unchanged even if the SA returns a different
 781                 * GID to use in the AH.
 782                 */
 783                if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
 784                           sizeof(union ib_gid))) {
 785                        ipoib_dbg(
 786                                priv,
 787                                "%s got PathRec for gid %pI6 while asked for %pI6\n",
 788                                dev->name, pathrec->dgid.raw,
 789                                path->pathrec.dgid.raw);
 790                        memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
 791                               sizeof(union ib_gid));
 792                }
 793
 794                path->pathrec = *pathrec;
 795
 796                old_ah   = path->ah;
 797                path->ah = ah;
 798
 799                ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
 800                          ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
 801                          pathrec->sl);
 802
 803                while ((skb = __skb_dequeue(&path->queue)))
 804                        __skb_queue_tail(&skqueue, skb);
 805
 806                list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
 807                        if (neigh->ah) {
 808                                WARN_ON(neigh->ah != old_ah);
 809                                /*
 810                                 * Dropping the ah reference inside
 811                                 * priv->lock is safe here, because we
 812                                 * will hold one more reference from
 813                                 * the original value of path->ah (ie
 814                                 * old_ah).
 815                                 */
 816                                ipoib_put_ah(neigh->ah);
 817                        }
 818                        kref_get(&path->ah->ref);
 819                        neigh->ah = path->ah;
 820
 821                        if (ipoib_cm_enabled(dev, neigh->daddr)) {
 822                                if (!ipoib_cm_get(neigh))
 823                                        ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,
 824                                                                               path,
 825                                                                               neigh));
 826                                if (!ipoib_cm_get(neigh)) {
 827                                        ipoib_neigh_free(neigh);
 828                                        continue;
 829                                }
 830                        }
 831
 832                        while ((skb = __skb_dequeue(&neigh->queue)))
 833                                __skb_queue_tail(&skqueue, skb);
 834                }
 835                path->ah->valid = 1;
 836        }
 837
 838        path->query = NULL;
 839        complete(&path->done);
 840
 841        spin_unlock_irqrestore(&priv->lock, flags);
 842
 843        if (IS_ERR_OR_NULL(ah))
 844                ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
 845
 846        if (old_ah)
 847                ipoib_put_ah(old_ah);
 848
 849        while ((skb = __skb_dequeue(&skqueue))) {
 850                int ret;
 851                skb->dev = dev;
 852                ret = dev_queue_xmit(skb);
 853                if (ret)
 854                        ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
 855                                   __func__, ret);
 856        }
 857}
 858
 859static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
 860                          void *gid)
 861{
 862        path->dev = priv->dev;
 863
 864        if (rdma_cap_opa_ah(priv->ca, priv->port))
 865                path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
 866        else
 867                path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
 868
 869        memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
 870        path->pathrec.sgid          = priv->local_gid;
 871        path->pathrec.pkey          = cpu_to_be16(priv->pkey);
 872        path->pathrec.numb_path     = 1;
 873        path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
 874}
 875
 876static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
 877{
 878        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 879        struct ipoib_path *path;
 880
 881        if (!priv->broadcast)
 882                return NULL;
 883
 884        path = kzalloc(sizeof(*path), GFP_ATOMIC);
 885        if (!path)
 886                return NULL;
 887
 888        skb_queue_head_init(&path->queue);
 889
 890        INIT_LIST_HEAD(&path->neigh_list);
 891
 892        init_path_rec(priv, path, gid);
 893
 894        return path;
 895}
 896
 897static int path_rec_start(struct net_device *dev,
 898                          struct ipoib_path *path)
 899{
 900        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 901
 902        ipoib_dbg(priv, "Start path record lookup for %pI6\n",
 903                  path->pathrec.dgid.raw);
 904
 905        init_completion(&path->done);
 906
 907        path->query_id =
 908                ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
 909                                   &path->pathrec,
 910                                   IB_SA_PATH_REC_DGID          |
 911                                   IB_SA_PATH_REC_SGID          |
 912                                   IB_SA_PATH_REC_NUMB_PATH     |
 913                                   IB_SA_PATH_REC_TRAFFIC_CLASS |
 914                                   IB_SA_PATH_REC_PKEY,
 915                                   1000, GFP_ATOMIC,
 916                                   path_rec_completion,
 917                                   path, &path->query);
 918        if (path->query_id < 0) {
 919                ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
 920                path->query = NULL;
 921                complete(&path->done);
 922                return path->query_id;
 923        }
 924
 925        return 0;
 926}
 927
 928static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
 929                               struct net_device *dev)
 930{
 931        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 932        struct ipoib_path *path;
 933        unsigned long flags;
 934
 935        spin_lock_irqsave(&priv->lock, flags);
 936
 937        path = __path_find(dev, daddr + 4);
 938        if (!path)
 939                goto out;
 940        if (!path->query)
 941                path_rec_start(dev, path);
 942out:
 943        spin_unlock_irqrestore(&priv->lock, flags);
 944}
 945
 946static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
 947                                          struct net_device *dev)
 948{
 949        struct ipoib_dev_priv *priv = ipoib_priv(dev);
 950        struct rdma_netdev *rn = netdev_priv(dev);
 951        struct ipoib_path *path;
 952        struct ipoib_neigh *neigh;
 953        unsigned long flags;
 954
 955        spin_lock_irqsave(&priv->lock, flags);
 956        neigh = ipoib_neigh_alloc(daddr, dev);
 957        if (!neigh) {
 958                spin_unlock_irqrestore(&priv->lock, flags);
 959                ++dev->stats.tx_dropped;
 960                dev_kfree_skb_any(skb);
 961                return NULL;
 962        }
 963
 964        /* To avoid race condition, make sure that the
 965         * neigh will be added only once.
 966         */
 967        if (unlikely(!list_empty(&neigh->list))) {
 968                spin_unlock_irqrestore(&priv->lock, flags);
 969                return neigh;
 970        }
 971
 972        path = __path_find(dev, daddr + 4);
 973        if (!path) {
 974                path = path_rec_create(dev, daddr + 4);
 975                if (!path)
 976                        goto err_path;
 977
 978                __path_add(dev, path);
 979        }
 980
 981        list_add_tail(&neigh->list, &path->neigh_list);
 982
 983        if (path->ah && path->ah->valid) {
 984                kref_get(&path->ah->ref);
 985                neigh->ah = path->ah;
 986
 987                if (ipoib_cm_enabled(dev, neigh->daddr)) {
 988                        if (!ipoib_cm_get(neigh))
 989                                ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
 990                        if (!ipoib_cm_get(neigh)) {
 991                                ipoib_neigh_free(neigh);
 992                                goto err_drop;
 993                        }
 994                        if (skb_queue_len(&neigh->queue) <
 995                            IPOIB_MAX_PATH_REC_QUEUE) {
 996                                push_pseudo_header(skb, neigh->daddr);
 997                                __skb_queue_tail(&neigh->queue, skb);
 998                        } else {
 999                                ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
1000                                           skb_queue_len(&neigh->queue));
1001                                goto err_drop;
1002                        }
1003                } else {
1004                        spin_unlock_irqrestore(&priv->lock, flags);
1005                        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1006                                                       IPOIB_QPN(daddr));
1007                        ipoib_neigh_put(neigh);
1008                        return NULL;
1009                }
1010        } else {
1011                neigh->ah  = NULL;
1012
1013                if (!path->query && path_rec_start(dev, path))
1014                        goto err_path;
1015                if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1016                        push_pseudo_header(skb, neigh->daddr);
1017                        __skb_queue_tail(&neigh->queue, skb);
1018                } else {
1019                        goto err_drop;
1020                }
1021        }
1022
1023        spin_unlock_irqrestore(&priv->lock, flags);
1024        ipoib_neigh_put(neigh);
1025        return NULL;
1026
1027err_path:
1028        ipoib_neigh_free(neigh);
1029err_drop:
1030        ++dev->stats.tx_dropped;
1031        dev_kfree_skb_any(skb);
1032
1033        spin_unlock_irqrestore(&priv->lock, flags);
1034        ipoib_neigh_put(neigh);
1035
1036        return NULL;
1037}
1038
1039static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
1040                             struct ipoib_pseudo_header *phdr)
1041{
1042        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1043        struct rdma_netdev *rn = netdev_priv(dev);
1044        struct ipoib_path *path;
1045        unsigned long flags;
1046
1047        spin_lock_irqsave(&priv->lock, flags);
1048
1049        /* no broadcast means that all paths are (going to be) not valid */
1050        if (!priv->broadcast)
1051                goto drop_and_unlock;
1052
1053        path = __path_find(dev, phdr->hwaddr + 4);
1054        if (!path || !path->ah || !path->ah->valid) {
1055                if (!path) {
1056                        path = path_rec_create(dev, phdr->hwaddr + 4);
1057                        if (!path)
1058                                goto drop_and_unlock;
1059                        __path_add(dev, path);
1060                } else {
1061                        /*
1062                         * make sure there are no changes in the existing
1063                         * path record
1064                         */
1065                        init_path_rec(priv, path, phdr->hwaddr + 4);
1066                }
1067                if (!path->query && path_rec_start(dev, path)) {
1068                        goto drop_and_unlock;
1069                }
1070
1071                if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1072                        push_pseudo_header(skb, phdr->hwaddr);
1073                        __skb_queue_tail(&path->queue, skb);
1074                        goto unlock;
1075                } else {
1076                        goto drop_and_unlock;
1077                }
1078        }
1079
1080        spin_unlock_irqrestore(&priv->lock, flags);
1081        ipoib_dbg(priv, "Send unicast ARP to %08x\n",
1082                  be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
1083        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1084                                       IPOIB_QPN(phdr->hwaddr));
1085        return;
1086
1087drop_and_unlock:
1088        ++dev->stats.tx_dropped;
1089        dev_kfree_skb_any(skb);
1090unlock:
1091        spin_unlock_irqrestore(&priv->lock, flags);
1092}
1093
1094static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
1095{
1096        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1097        struct rdma_netdev *rn = netdev_priv(dev);
1098        struct ipoib_neigh *neigh;
1099        struct ipoib_pseudo_header *phdr;
1100        struct ipoib_header *header;
1101        unsigned long flags;
1102
1103        phdr = (struct ipoib_pseudo_header *) skb->data;
1104        skb_pull(skb, sizeof(*phdr));
1105        header = (struct ipoib_header *) skb->data;
1106
1107        if (unlikely(phdr->hwaddr[4] == 0xff)) {
1108                /* multicast, arrange "if" according to probability */
1109                if ((header->proto != htons(ETH_P_IP)) &&
1110                    (header->proto != htons(ETH_P_IPV6)) &&
1111                    (header->proto != htons(ETH_P_ARP)) &&
1112                    (header->proto != htons(ETH_P_RARP)) &&
1113                    (header->proto != htons(ETH_P_TIPC))) {
1114                        /* ethertype not supported by IPoIB */
1115                        ++dev->stats.tx_dropped;
1116                        dev_kfree_skb_any(skb);
1117                        return NETDEV_TX_OK;
1118                }
1119                /* Add in the P_Key for multicast*/
1120                phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
1121                phdr->hwaddr[9] = priv->pkey & 0xff;
1122
1123                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
1124                if (likely(neigh))
1125                        goto send_using_neigh;
1126                ipoib_mcast_send(dev, phdr->hwaddr, skb);
1127                return NETDEV_TX_OK;
1128        }
1129
1130        /* unicast, arrange "switch" according to probability */
1131        switch (header->proto) {
1132        case htons(ETH_P_IP):
1133        case htons(ETH_P_IPV6):
1134        case htons(ETH_P_TIPC):
1135                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
1136                if (unlikely(!neigh)) {
1137                        neigh = neigh_add_path(skb, phdr->hwaddr, dev);
1138                        if (likely(!neigh))
1139                                return NETDEV_TX_OK;
1140                }
1141                break;
1142        case htons(ETH_P_ARP):
1143        case htons(ETH_P_RARP):
1144                /* for unicast ARP and RARP should always perform path find */
1145                unicast_arp_send(skb, dev, phdr);
1146                return NETDEV_TX_OK;
1147        default:
1148                /* ethertype not supported by IPoIB */
1149                ++dev->stats.tx_dropped;
1150                dev_kfree_skb_any(skb);
1151                return NETDEV_TX_OK;
1152        }
1153
1154send_using_neigh:
1155        /* note we now hold a ref to neigh */
1156        if (ipoib_cm_get(neigh)) {
1157                if (ipoib_cm_up(neigh)) {
1158                        ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
1159                        goto unref;
1160                }
1161        } else if (neigh->ah && neigh->ah->valid) {
1162                neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
1163                                                IPOIB_QPN(phdr->hwaddr));
1164                goto unref;
1165        } else if (neigh->ah) {
1166                neigh_refresh_path(neigh, phdr->hwaddr, dev);
1167        }
1168
1169        if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1170                push_pseudo_header(skb, phdr->hwaddr);
1171                spin_lock_irqsave(&priv->lock, flags);
1172                __skb_queue_tail(&neigh->queue, skb);
1173                spin_unlock_irqrestore(&priv->lock, flags);
1174        } else {
1175                ++dev->stats.tx_dropped;
1176                dev_kfree_skb_any(skb);
1177        }
1178
1179unref:
1180        ipoib_neigh_put(neigh);
1181
1182        return NETDEV_TX_OK;
1183}
1184
1185static void ipoib_timeout(struct net_device *dev)
1186{
1187        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1188
1189        ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
1190                   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
1191        ipoib_warn(priv, "queue stopped %d, tx_head %u, tx_tail %u\n",
1192                   netif_queue_stopped(dev),
1193                   priv->tx_head, priv->tx_tail);
1194        /* XXX reset QP, etc. */
1195}
1196
1197static int ipoib_hard_header(struct sk_buff *skb,
1198                             struct net_device *dev,
1199                             unsigned short type,
1200                             const void *daddr,
1201                             const void *saddr,
1202                             unsigned int len)
1203{
1204        struct ipoib_header *header;
1205
1206        header = skb_push(skb, sizeof(*header));
1207
1208        header->proto = htons(type);
1209        header->reserved = 0;
1210
1211        /*
1212         * we don't rely on dst_entry structure,  always stuff the
1213         * destination address into skb hard header so we can figure out where
1214         * to send the packet later.
1215         */
1216        push_pseudo_header(skb, daddr);
1217
1218        return IPOIB_HARD_LEN;
1219}
1220
1221static void ipoib_set_mcast_list(struct net_device *dev)
1222{
1223        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1224
1225        if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
1226                ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
1227                return;
1228        }
1229
1230        queue_work(priv->wq, &priv->restart_task);
1231}
1232
1233static int ipoib_get_iflink(const struct net_device *dev)
1234{
1235        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1236
1237        /* parent interface */
1238        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
1239                return dev->ifindex;
1240
1241        /* child/vlan interface */
1242        return priv->parent->ifindex;
1243}
1244
1245static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
1246{
1247        /*
1248         * Use only the address parts that contributes to spreading
1249         * The subnet prefix is not used as one can not connect to
1250         * same remote port (GUID) using the same remote QPN via two
1251         * different subnets.
1252         */
1253         /* qpn octets[1:4) & port GUID octets[12:20) */
1254        u32 *d32 = (u32 *) daddr;
1255        u32 hv;
1256
1257        hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);
1258        return hv & htbl->mask;
1259}
1260
1261struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
1262{
1263        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1264        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1265        struct ipoib_neigh_hash *htbl;
1266        struct ipoib_neigh *neigh = NULL;
1267        u32 hash_val;
1268
1269        rcu_read_lock_bh();
1270
1271        htbl = rcu_dereference_bh(ntbl->htbl);
1272
1273        if (!htbl)
1274                goto out_unlock;
1275
1276        hash_val = ipoib_addr_hash(htbl, daddr);
1277        for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
1278             neigh != NULL;
1279             neigh = rcu_dereference_bh(neigh->hnext)) {
1280                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1281                        /* found, take one ref on behalf of the caller */
1282                        if (!atomic_inc_not_zero(&neigh->refcnt)) {
1283                                /* deleted */
1284                                neigh = NULL;
1285                                goto out_unlock;
1286                        }
1287
1288                        if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
1289                                neigh->alive = jiffies;
1290                        goto out_unlock;
1291                }
1292        }
1293
1294out_unlock:
1295        rcu_read_unlock_bh();
1296        return neigh;
1297}
1298
1299static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
1300{
1301        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1302        struct ipoib_neigh_hash *htbl;
1303        unsigned long neigh_obsolete;
1304        unsigned long dt;
1305        unsigned long flags;
1306        int i;
1307        LIST_HEAD(remove_list);
1308
1309        spin_lock_irqsave(&priv->lock, flags);
1310
1311        htbl = rcu_dereference_protected(ntbl->htbl,
1312                                         lockdep_is_held(&priv->lock));
1313
1314        if (!htbl)
1315                goto out_unlock;
1316
1317        /* neigh is obsolete if it was idle for two GC periods */
1318        dt = 2 * arp_tbl.gc_interval;
1319        neigh_obsolete = jiffies - dt;
1320
1321        for (i = 0; i < htbl->size; i++) {
1322                struct ipoib_neigh *neigh;
1323                struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1324
1325                while ((neigh = rcu_dereference_protected(*np,
1326                                                          lockdep_is_held(&priv->lock))) != NULL) {
1327                        /* was the neigh idle for two GC periods */
1328                        if (time_after(neigh_obsolete, neigh->alive)) {
1329
1330                                ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);
1331
1332                                rcu_assign_pointer(*np,
1333                                                   rcu_dereference_protected(neigh->hnext,
1334                                                                             lockdep_is_held(&priv->lock)));
1335                                /* remove from path/mc list */
1336                                list_del_init(&neigh->list);
1337                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1338                        } else {
1339                                np = &neigh->hnext;
1340                        }
1341
1342                }
1343        }
1344
1345out_unlock:
1346        spin_unlock_irqrestore(&priv->lock, flags);
1347        ipoib_mcast_remove_list(&remove_list);
1348}
1349
1350static void ipoib_reap_neigh(struct work_struct *work)
1351{
1352        struct ipoib_dev_priv *priv =
1353                container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);
1354
1355        __ipoib_reap_neigh(priv);
1356
1357        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
1358                           arp_tbl.gc_interval);
1359}
1360
1361
1362static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
1363                                      struct net_device *dev)
1364{
1365        struct ipoib_neigh *neigh;
1366
1367        neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);
1368        if (!neigh)
1369                return NULL;
1370
1371        neigh->dev = dev;
1372        memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
1373        skb_queue_head_init(&neigh->queue);
1374        INIT_LIST_HEAD(&neigh->list);
1375        ipoib_cm_set(neigh, NULL);
1376        /* one ref on behalf of the caller */
1377        atomic_set(&neigh->refcnt, 1);
1378
1379        return neigh;
1380}
1381
1382struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
1383                                      struct net_device *dev)
1384{
1385        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1386        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1387        struct ipoib_neigh_hash *htbl;
1388        struct ipoib_neigh *neigh;
1389        u32 hash_val;
1390
1391        htbl = rcu_dereference_protected(ntbl->htbl,
1392                                         lockdep_is_held(&priv->lock));
1393        if (!htbl) {
1394                neigh = NULL;
1395                goto out_unlock;
1396        }
1397
1398        /* need to add a new neigh, but maybe some other thread succeeded?
1399         * recalc hash, maybe hash resize took place so we do a search
1400         */
1401        hash_val = ipoib_addr_hash(htbl, daddr);
1402        for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
1403                                               lockdep_is_held(&priv->lock));
1404             neigh != NULL;
1405             neigh = rcu_dereference_protected(neigh->hnext,
1406                                               lockdep_is_held(&priv->lock))) {
1407                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1408                        /* found, take one ref on behalf of the caller */
1409                        if (!atomic_inc_not_zero(&neigh->refcnt)) {
1410                                /* deleted */
1411                                neigh = NULL;
1412                                break;
1413                        }
1414                        neigh->alive = jiffies;
1415                        goto out_unlock;
1416                }
1417        }
1418
1419        neigh = ipoib_neigh_ctor(daddr, dev);
1420        if (!neigh)
1421                goto out_unlock;
1422
1423        /* one ref on behalf of the hash table */
1424        atomic_inc(&neigh->refcnt);
1425        neigh->alive = jiffies;
1426        /* put in hash */
1427        rcu_assign_pointer(neigh->hnext,
1428                           rcu_dereference_protected(htbl->buckets[hash_val],
1429                                                     lockdep_is_held(&priv->lock)));
1430        rcu_assign_pointer(htbl->buckets[hash_val], neigh);
1431        atomic_inc(&ntbl->entries);
1432
1433out_unlock:
1434
1435        return neigh;
1436}
1437
1438void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
1439{
1440        /* neigh reference count was dropprd to zero */
1441        struct net_device *dev = neigh->dev;
1442        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1443        struct sk_buff *skb;
1444        if (neigh->ah)
1445                ipoib_put_ah(neigh->ah);
1446        while ((skb = __skb_dequeue(&neigh->queue))) {
1447                ++dev->stats.tx_dropped;
1448                dev_kfree_skb_any(skb);
1449        }
1450        if (ipoib_cm_get(neigh))
1451                ipoib_cm_destroy_tx(ipoib_cm_get(neigh));
1452        ipoib_dbg(ipoib_priv(dev),
1453                  "neigh free for %06x %pI6\n",
1454                  IPOIB_QPN(neigh->daddr),
1455                  neigh->daddr + 4);
1456        kfree(neigh);
1457        if (atomic_dec_and_test(&priv->ntbl.entries)) {
1458                if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
1459                        complete(&priv->ntbl.flushed);
1460        }
1461}
1462
1463static void ipoib_neigh_reclaim(struct rcu_head *rp)
1464{
1465        /* Called as a result of removal from hash table */
1466        struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
1467        /* note TX context may hold another ref */
1468        ipoib_neigh_put(neigh);
1469}
1470
1471void ipoib_neigh_free(struct ipoib_neigh *neigh)
1472{
1473        struct net_device *dev = neigh->dev;
1474        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1475        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1476        struct ipoib_neigh_hash *htbl;
1477        struct ipoib_neigh __rcu **np;
1478        struct ipoib_neigh *n;
1479        u32 hash_val;
1480
1481        htbl = rcu_dereference_protected(ntbl->htbl,
1482                                        lockdep_is_held(&priv->lock));
1483        if (!htbl)
1484                return;
1485
1486        hash_val = ipoib_addr_hash(htbl, neigh->daddr);
1487        np = &htbl->buckets[hash_val];
1488        for (n = rcu_dereference_protected(*np,
1489                                            lockdep_is_held(&priv->lock));
1490             n != NULL;
1491             n = rcu_dereference_protected(*np,
1492                                        lockdep_is_held(&priv->lock))) {
1493                if (n == neigh) {
1494                        /* found */
1495                        rcu_assign_pointer(*np,
1496                                           rcu_dereference_protected(neigh->hnext,
1497                                                                     lockdep_is_held(&priv->lock)));
1498                        /* remove from parent list */
1499                        list_del_init(&neigh->list);
1500                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1501                        return;
1502                } else {
1503                        np = &n->hnext;
1504                }
1505        }
1506}
1507
1508static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
1509{
1510        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1511        struct ipoib_neigh_hash *htbl;
1512        struct ipoib_neigh __rcu **buckets;
1513        u32 size;
1514
1515        clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
1516        ntbl->htbl = NULL;
1517        htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);
1518        if (!htbl)
1519                return -ENOMEM;
1520        size = roundup_pow_of_two(arp_tbl.gc_thresh3);
1521        buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);
1522        if (!buckets) {
1523                kfree(htbl);
1524                return -ENOMEM;
1525        }
1526        htbl->size = size;
1527        htbl->mask = (size - 1);
1528        htbl->buckets = buckets;
1529        RCU_INIT_POINTER(ntbl->htbl, htbl);
1530        htbl->ntbl = ntbl;
1531        atomic_set(&ntbl->entries, 0);
1532
1533        /* start garbage collection */
1534        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
1535                           arp_tbl.gc_interval);
1536
1537        return 0;
1538}
1539
1540static void neigh_hash_free_rcu(struct rcu_head *head)
1541{
1542        struct ipoib_neigh_hash *htbl = container_of(head,
1543                                                    struct ipoib_neigh_hash,
1544                                                    rcu);
1545        struct ipoib_neigh __rcu **buckets = htbl->buckets;
1546        struct ipoib_neigh_table *ntbl = htbl->ntbl;
1547
1548        kvfree(buckets);
1549        kfree(htbl);
1550        complete(&ntbl->deleted);
1551}
1552
1553void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
1554{
1555        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1556        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1557        struct ipoib_neigh_hash *htbl;
1558        unsigned long flags;
1559        int i;
1560
1561        /* remove all neigh connected to a given path or mcast */
1562        spin_lock_irqsave(&priv->lock, flags);
1563
1564        htbl = rcu_dereference_protected(ntbl->htbl,
1565                                         lockdep_is_held(&priv->lock));
1566
1567        if (!htbl)
1568                goto out_unlock;
1569
1570        for (i = 0; i < htbl->size; i++) {
1571                struct ipoib_neigh *neigh;
1572                struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1573
1574                while ((neigh = rcu_dereference_protected(*np,
1575                                                          lockdep_is_held(&priv->lock))) != NULL) {
1576                        /* delete neighs belong to this parent */
1577                        if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {
1578                                rcu_assign_pointer(*np,
1579                                                   rcu_dereference_protected(neigh->hnext,
1580                                                                             lockdep_is_held(&priv->lock)));
1581                                /* remove from parent list */
1582                                list_del_init(&neigh->list);
1583                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1584                        } else {
1585                                np = &neigh->hnext;
1586                        }
1587
1588                }
1589        }
1590out_unlock:
1591        spin_unlock_irqrestore(&priv->lock, flags);
1592}
1593
1594static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
1595{
1596        struct ipoib_neigh_table *ntbl = &priv->ntbl;
1597        struct ipoib_neigh_hash *htbl;
1598        unsigned long flags;
1599        int i, wait_flushed = 0;
1600
1601        init_completion(&priv->ntbl.flushed);
1602        set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
1603
1604        spin_lock_irqsave(&priv->lock, flags);
1605
1606        htbl = rcu_dereference_protected(ntbl->htbl,
1607                                        lockdep_is_held(&priv->lock));
1608        if (!htbl)
1609                goto out_unlock;
1610
1611        wait_flushed = atomic_read(&priv->ntbl.entries);
1612        if (!wait_flushed)
1613                goto free_htbl;
1614
1615        for (i = 0; i < htbl->size; i++) {
1616                struct ipoib_neigh *neigh;
1617                struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1618
1619                while ((neigh = rcu_dereference_protected(*np,
1620                                       lockdep_is_held(&priv->lock))) != NULL) {
1621                        rcu_assign_pointer(*np,
1622                                           rcu_dereference_protected(neigh->hnext,
1623                                                                     lockdep_is_held(&priv->lock)));
1624                        /* remove from path/mc list */
1625                        list_del_init(&neigh->list);
1626                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1627                }
1628        }
1629
1630free_htbl:
1631        rcu_assign_pointer(ntbl->htbl, NULL);
1632        call_rcu(&htbl->rcu, neigh_hash_free_rcu);
1633
1634out_unlock:
1635        spin_unlock_irqrestore(&priv->lock, flags);
1636        if (wait_flushed)
1637                wait_for_completion(&priv->ntbl.flushed);
1638}
1639
1640static void ipoib_neigh_hash_uninit(struct net_device *dev)
1641{
1642        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1643
1644        ipoib_dbg(priv, "%s\n", __func__);
1645        init_completion(&priv->ntbl.deleted);
1646
1647        cancel_delayed_work_sync(&priv->neigh_reap_task);
1648
1649        ipoib_flush_neighs(priv);
1650
1651        wait_for_completion(&priv->ntbl.deleted);
1652}
1653
1654static void ipoib_napi_add(struct net_device *dev)
1655{
1656        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1657
1658        netif_napi_add(dev, &priv->recv_napi, ipoib_rx_poll, IPOIB_NUM_WC);
1659        netif_napi_add(dev, &priv->send_napi, ipoib_tx_poll, MAX_SEND_CQE);
1660}
1661
1662static void ipoib_napi_del(struct net_device *dev)
1663{
1664        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1665
1666        netif_napi_del(&priv->recv_napi);
1667        netif_napi_del(&priv->send_napi);
1668}
1669
1670static void ipoib_dev_uninit_default(struct net_device *dev)
1671{
1672        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1673
1674        ipoib_transport_dev_cleanup(dev);
1675
1676        ipoib_napi_del(dev);
1677
1678        ipoib_cm_dev_cleanup(dev);
1679
1680        kfree(priv->rx_ring);
1681        vfree(priv->tx_ring);
1682
1683        priv->rx_ring = NULL;
1684        priv->tx_ring = NULL;
1685}
1686
1687static int ipoib_dev_init_default(struct net_device *dev)
1688{
1689        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1690
1691        ipoib_napi_add(dev);
1692
1693        /* Allocate RX/TX "rings" to hold queued skbs */
1694        priv->rx_ring = kcalloc(ipoib_recvq_size,
1695                                       sizeof(*priv->rx_ring),
1696                                       GFP_KERNEL);
1697        if (!priv->rx_ring)
1698                goto out;
1699
1700        priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
1701                                           sizeof(*priv->tx_ring)));
1702        if (!priv->tx_ring) {
1703                pr_warn("%s: failed to allocate TX ring (%d entries)\n",
1704                        priv->ca->name, ipoib_sendq_size);
1705                goto out_rx_ring_cleanup;
1706        }
1707
1708        /* priv->tx_head, tx_tail & tx_outstanding are already 0 */
1709
1710        if (ipoib_transport_dev_init(dev, priv->ca)) {
1711                pr_warn("%s: ipoib_transport_dev_init failed\n",
1712                        priv->ca->name);
1713                goto out_tx_ring_cleanup;
1714        }
1715
1716        /* after qp created set dev address */
1717        priv->dev->dev_addr[1] = (priv->qp->qp_num >> 16) & 0xff;
1718        priv->dev->dev_addr[2] = (priv->qp->qp_num >>  8) & 0xff;
1719        priv->dev->dev_addr[3] = (priv->qp->qp_num) & 0xff;
1720
1721        return 0;
1722
1723out_tx_ring_cleanup:
1724        vfree(priv->tx_ring);
1725
1726out_rx_ring_cleanup:
1727        kfree(priv->rx_ring);
1728
1729out:
1730        ipoib_napi_del(dev);
1731        return -ENOMEM;
1732}
1733
1734static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
1735                       int cmd)
1736{
1737        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1738
1739        if (!priv->rn_ops->ndo_do_ioctl)
1740                return -EOPNOTSUPP;
1741
1742        return priv->rn_ops->ndo_do_ioctl(dev, ifr, cmd);
1743}
1744
1745static int ipoib_dev_init(struct net_device *dev)
1746{
1747        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1748        int ret = -ENOMEM;
1749
1750        priv->qp = NULL;
1751
1752        /*
1753         * the various IPoIB tasks assume they will never race against
1754         * themselves, so always use a single thread workqueue
1755         */
1756        priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
1757        if (!priv->wq) {
1758                pr_warn("%s: failed to allocate device WQ\n", dev->name);
1759                goto out;
1760        }
1761
1762        /* create pd, which used both for control and datapath*/
1763        priv->pd = ib_alloc_pd(priv->ca, 0);
1764        if (IS_ERR(priv->pd)) {
1765                pr_warn("%s: failed to allocate PD\n", priv->ca->name);
1766                goto clean_wq;
1767        }
1768
1769        ret = priv->rn_ops->ndo_init(dev);
1770        if (ret) {
1771                pr_warn("%s failed to init HW resource\n", dev->name);
1772                goto out_free_pd;
1773        }
1774
1775        ret = ipoib_neigh_hash_init(priv);
1776        if (ret) {
1777                pr_warn("%s failed to init neigh hash\n", dev->name);
1778                goto out_dev_uninit;
1779        }
1780
1781        if (dev->flags & IFF_UP) {
1782                if (ipoib_ib_dev_open(dev)) {
1783                        pr_warn("%s failed to open device\n", dev->name);
1784                        ret = -ENODEV;
1785                        goto out_hash_uninit;
1786                }
1787        }
1788
1789        return 0;
1790
1791out_hash_uninit:
1792        ipoib_neigh_hash_uninit(dev);
1793
1794out_dev_uninit:
1795        ipoib_ib_dev_cleanup(dev);
1796
1797out_free_pd:
1798        if (priv->pd) {
1799                ib_dealloc_pd(priv->pd);
1800                priv->pd = NULL;
1801        }
1802
1803clean_wq:
1804        if (priv->wq) {
1805                destroy_workqueue(priv->wq);
1806                priv->wq = NULL;
1807        }
1808
1809out:
1810        return ret;
1811}
1812
1813/*
1814 * This must be called before doing an unregister_netdev on a parent device to
1815 * shutdown the IB event handler.
1816 */
1817static void ipoib_parent_unregister_pre(struct net_device *ndev)
1818{
1819        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1820
1821        /*
1822         * ipoib_set_mac checks netif_running before pushing work, clearing
1823         * running ensures the it will not add more work.
1824         */
1825        rtnl_lock();
1826        dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);
1827        rtnl_unlock();
1828
1829        /* ipoib_event() cannot be running once this returns */
1830        ib_unregister_event_handler(&priv->event_handler);
1831
1832        /*
1833         * Work on the queue grabs the rtnl lock, so this cannot be done while
1834         * also holding it.
1835         */
1836        flush_workqueue(ipoib_workqueue);
1837}
1838
1839static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
1840{
1841        priv->hca_caps = priv->ca->attrs.device_cap_flags;
1842
1843        if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
1844                priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
1845
1846                if (priv->hca_caps & IB_DEVICE_UD_TSO)
1847                        priv->dev->hw_features |= NETIF_F_TSO;
1848
1849                priv->dev->features |= priv->dev->hw_features;
1850        }
1851}
1852
1853static int ipoib_parent_init(struct net_device *ndev)
1854{
1855        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1856        struct ib_port_attr attr;
1857        int result;
1858
1859        result = ib_query_port(priv->ca, priv->port, &attr);
1860        if (result) {
1861                pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
1862                        priv->port);
1863                return result;
1864        }
1865        priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu);
1866
1867        result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
1868        if (result) {
1869                pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
1870                        priv->ca->name, priv->port, result);
1871                return result;
1872        }
1873
1874        result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);
1875        if (result) {
1876                pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
1877                        priv->ca->name, priv->port, result);
1878                return result;
1879        }
1880        memcpy(priv->dev->dev_addr + 4, priv->local_gid.raw,
1881               sizeof(union ib_gid));
1882
1883        SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
1884        priv->dev->dev_port = priv->port - 1;
1885        /* Let's set this one too for backwards compatibility. */
1886        priv->dev->dev_id = priv->port - 1;
1887
1888        return 0;
1889}
1890
1891static void ipoib_child_init(struct net_device *ndev)
1892{
1893        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1894        struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
1895
1896        dev_hold(priv->parent);
1897
1898        down_write(&ppriv->vlan_rwsem);
1899        list_add_tail(&priv->list, &ppriv->child_intfs);
1900        up_write(&ppriv->vlan_rwsem);
1901
1902        priv->max_ib_mtu = ppriv->max_ib_mtu;
1903        set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
1904        memcpy(priv->dev->dev_addr, ppriv->dev->dev_addr, INFINIBAND_ALEN);
1905        memcpy(&priv->local_gid, &ppriv->local_gid, sizeof(priv->local_gid));
1906}
1907
1908static int ipoib_ndo_init(struct net_device *ndev)
1909{
1910        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1911        int rc;
1912
1913        if (priv->parent) {
1914                ipoib_child_init(ndev);
1915        } else {
1916                rc = ipoib_parent_init(ndev);
1917                if (rc)
1918                        return rc;
1919        }
1920
1921        /* MTU will be reset when mcast join happens */
1922        ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
1923        priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
1924        ndev->max_mtu = IPOIB_CM_MTU;
1925
1926        ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
1927
1928        /*
1929         * Set the full membership bit, so that we join the right
1930         * broadcast group, etc.
1931         */
1932        priv->pkey |= 0x8000;
1933
1934        ndev->broadcast[8] = priv->pkey >> 8;
1935        ndev->broadcast[9] = priv->pkey & 0xff;
1936        set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
1937
1938        ipoib_set_dev_features(priv);
1939
1940        rc = ipoib_dev_init(ndev);
1941        if (rc) {
1942                pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
1943                        priv->ca->name, priv->dev->name, priv->port, rc);
1944        }
1945
1946        return 0;
1947}
1948
1949static void ipoib_ndo_uninit(struct net_device *dev)
1950{
1951        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1952
1953        ASSERT_RTNL();
1954
1955        /*
1956         * ipoib_remove_one guarantees the children are removed before the
1957         * parent, and that is the only place where a parent can be removed.
1958         */
1959        WARN_ON(!list_empty(&priv->child_intfs));
1960
1961        ipoib_neigh_hash_uninit(dev);
1962
1963        ipoib_ib_dev_cleanup(dev);
1964
1965        /* no more works over the priv->wq */
1966        if (priv->wq) {
1967                flush_workqueue(priv->wq);
1968                destroy_workqueue(priv->wq);
1969                priv->wq = NULL;
1970        }
1971
1972        if (priv->parent) {
1973                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
1974
1975                down_write(&ppriv->vlan_rwsem);
1976                list_del(&priv->list);
1977                up_write(&ppriv->vlan_rwsem);
1978
1979                dev_put(priv->parent);
1980        }
1981}
1982
1983static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
1984{
1985        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1986
1987        return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);
1988}
1989
1990static int ipoib_get_vf_config(struct net_device *dev, int vf,
1991                               struct ifla_vf_info *ivf)
1992{
1993        struct ipoib_dev_priv *priv = ipoib_priv(dev);
1994        int err;
1995
1996        err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);
1997        if (err)
1998                return err;
1999
2000        ivf->vf = vf;
2001
2002        return 0;
2003}
2004
2005static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
2006{
2007        struct ipoib_dev_priv *priv = ipoib_priv(dev);
2008
2009        if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
2010                return -EINVAL;
2011
2012        return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);
2013}
2014
2015static int ipoib_get_vf_stats(struct net_device *dev, int vf,
2016                              struct ifla_vf_stats *vf_stats)
2017{
2018        struct ipoib_dev_priv *priv = ipoib_priv(dev);
2019
2020        return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);
2021}
2022
2023static const struct header_ops ipoib_header_ops = {
2024        .create = ipoib_hard_header,
2025};
2026
2027static const struct net_device_ops ipoib_netdev_ops_pf = {
2028        .ndo_init                = ipoib_ndo_init,
2029        .ndo_uninit              = ipoib_ndo_uninit,
2030        .ndo_open                = ipoib_open,
2031        .ndo_stop                = ipoib_stop,
2032        .ndo_change_mtu          = ipoib_change_mtu,
2033        .ndo_fix_features        = ipoib_fix_features,
2034        .ndo_start_xmit          = ipoib_start_xmit,
2035        .ndo_tx_timeout          = ipoib_timeout,
2036        .ndo_set_rx_mode         = ipoib_set_mcast_list,
2037        .ndo_get_iflink          = ipoib_get_iflink,
2038        .ndo_set_vf_link_state   = ipoib_set_vf_link_state,
2039        .ndo_get_vf_config       = ipoib_get_vf_config,
2040        .ndo_get_vf_stats        = ipoib_get_vf_stats,
2041        .ndo_set_vf_guid         = ipoib_set_vf_guid,
2042        .ndo_set_mac_address     = ipoib_set_mac,
2043        .ndo_get_stats64         = ipoib_get_stats,
2044        .ndo_do_ioctl            = ipoib_ioctl,
2045};
2046
2047static const struct net_device_ops ipoib_netdev_ops_vf = {
2048        .ndo_init                = ipoib_ndo_init,
2049        .ndo_uninit              = ipoib_ndo_uninit,
2050        .ndo_open                = ipoib_open,
2051        .ndo_stop                = ipoib_stop,
2052        .ndo_change_mtu          = ipoib_change_mtu,
2053        .ndo_fix_features        = ipoib_fix_features,
2054        .ndo_start_xmit          = ipoib_start_xmit,
2055        .ndo_tx_timeout          = ipoib_timeout,
2056        .ndo_set_rx_mode         = ipoib_set_mcast_list,
2057        .ndo_get_iflink          = ipoib_get_iflink,
2058        .ndo_get_stats64         = ipoib_get_stats,
2059        .ndo_do_ioctl            = ipoib_ioctl,
2060};
2061
2062void ipoib_setup_common(struct net_device *dev)
2063{
2064        dev->header_ops          = &ipoib_header_ops;
2065
2066        ipoib_set_ethtool_ops(dev);
2067
2068        dev->watchdog_timeo      = HZ;
2069
2070        dev->flags              |= IFF_BROADCAST | IFF_MULTICAST;
2071
2072        dev->hard_header_len     = IPOIB_HARD_LEN;
2073        dev->addr_len            = INFINIBAND_ALEN;
2074        dev->type                = ARPHRD_INFINIBAND;
2075        dev->tx_queue_len        = ipoib_sendq_size * 2;
2076        dev->features            = (NETIF_F_VLAN_CHALLENGED     |
2077                                    NETIF_F_HIGHDMA);
2078        netif_keep_dst(dev);
2079
2080        memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);
2081
2082        /*
2083         * unregister_netdev always frees the netdev, we use this mode
2084         * consistently to unify all the various unregister paths, including
2085         * those connected to rtnl_link_ops which require it.
2086         */
2087        dev->needs_free_netdev = true;
2088}
2089
2090static void ipoib_build_priv(struct net_device *dev)
2091{
2092        struct ipoib_dev_priv *priv = ipoib_priv(dev);
2093
2094        priv->dev = dev;
2095        spin_lock_init(&priv->lock);
2096        init_rwsem(&priv->vlan_rwsem);
2097        mutex_init(&priv->mcast_mutex);
2098
2099        INIT_LIST_HEAD(&priv->path_list);
2100        INIT_LIST_HEAD(&priv->child_intfs);
2101        INIT_LIST_HEAD(&priv->dead_ahs);
2102        INIT_LIST_HEAD(&priv->multicast_list);
2103
2104        INIT_DELAYED_WORK(&priv->mcast_task,   ipoib_mcast_join_task);
2105        INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
2106        INIT_WORK(&priv->flush_light,   ipoib_ib_dev_flush_light);
2107        INIT_WORK(&priv->flush_normal,   ipoib_ib_dev_flush_normal);
2108        INIT_WORK(&priv->flush_heavy,   ipoib_ib_dev_flush_heavy);
2109        INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
2110        INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
2111        INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
2112}
2113
2114static const struct net_device_ops ipoib_netdev_default_pf = {
2115        .ndo_init                = ipoib_dev_init_default,
2116        .ndo_uninit              = ipoib_dev_uninit_default,
2117        .ndo_open                = ipoib_ib_dev_open_default,
2118        .ndo_stop                = ipoib_ib_dev_stop_default,
2119};
2120
2121static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u8 port,
2122                                             const char *name)
2123{
2124        struct net_device *dev;
2125
2126        dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2127                                NET_NAME_UNKNOWN, ipoib_setup_common);
2128        if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)
2129                return dev;
2130
2131        dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
2132                           ipoib_setup_common);
2133        if (!dev)
2134                return ERR_PTR(-ENOMEM);
2135        return dev;
2136}
2137
2138int ipoib_intf_init(struct ib_device *hca, u8 port, const char *name,
2139                    struct net_device *dev)
2140{
2141        struct rdma_netdev *rn = netdev_priv(dev);
2142        struct ipoib_dev_priv *priv;
2143        int rc;
2144
2145        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2146        if (!priv)
2147                return -ENOMEM;
2148
2149        priv->ca = hca;
2150        priv->port = port;
2151
2152        rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2153                              NET_NAME_UNKNOWN, ipoib_setup_common, dev);
2154        if (rc) {
2155                if (rc != -EOPNOTSUPP)
2156                        goto out;
2157
2158                dev->netdev_ops = &ipoib_netdev_default_pf;
2159                rn->send = ipoib_send;
2160                rn->attach_mcast = ipoib_mcast_attach;
2161                rn->detach_mcast = ipoib_mcast_detach;
2162                rn->hca = hca;
2163        }
2164
2165        priv->rn_ops = dev->netdev_ops;
2166
2167        if (hca->attrs.device_cap_flags & IB_DEVICE_VIRTUAL_FUNCTION)
2168                dev->netdev_ops = &ipoib_netdev_ops_vf;
2169        else
2170                dev->netdev_ops = &ipoib_netdev_ops_pf;
2171
2172        rn->clnt_priv = priv;
2173        /*
2174         * Only the child register_netdev flows can handle priv_destructor
2175         * being set, so we force it to NULL here and handle manually until it
2176         * is safe to turn on.
2177         */
2178        priv->next_priv_destructor = dev->priv_destructor;
2179        dev->priv_destructor = NULL;
2180
2181        ipoib_build_priv(dev);
2182
2183        return 0;
2184
2185out:
2186        kfree(priv);
2187        return rc;
2188}
2189
2190struct net_device *ipoib_intf_alloc(struct ib_device *hca, u8 port,
2191                                    const char *name)
2192{
2193        struct net_device *dev;
2194        int rc;
2195
2196        dev = ipoib_alloc_netdev(hca, port, name);
2197        if (IS_ERR(dev))
2198                return dev;
2199
2200        rc = ipoib_intf_init(hca, port, name, dev);
2201        if (rc) {
2202                free_netdev(dev);
2203                return ERR_PTR(rc);
2204        }
2205
2206        /*
2207         * Upon success the caller must ensure ipoib_intf_free is called or
2208         * register_netdevice succeed'd and priv_destructor is set to
2209         * ipoib_intf_free.
2210         */
2211        return dev;
2212}
2213
2214void ipoib_intf_free(struct net_device *dev)
2215{
2216        struct ipoib_dev_priv *priv = ipoib_priv(dev);
2217        struct rdma_netdev *rn = netdev_priv(dev);
2218
2219        dev->priv_destructor = priv->next_priv_destructor;
2220        if (dev->priv_destructor)
2221                dev->priv_destructor(dev);
2222
2223        /*
2224         * There are some error flows around register_netdev failing that may
2225         * attempt to call priv_destructor twice, prevent that from happening.
2226         */
2227        dev->priv_destructor = NULL;
2228
2229        /* unregister/destroy is very complicated. Make bugs more obvious. */
2230        rn->clnt_priv = NULL;
2231
2232        kfree(priv);
2233}
2234
2235static ssize_t show_pkey(struct device *dev,
2236                         struct device_attribute *attr, char *buf)
2237{
2238        struct net_device *ndev = to_net_dev(dev);
2239        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2240
2241        return sprintf(buf, "0x%04x\n", priv->pkey);
2242}
2243static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
2244
2245static ssize_t show_umcast(struct device *dev,
2246                           struct device_attribute *attr, char *buf)
2247{
2248        struct net_device *ndev = to_net_dev(dev);
2249        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2250
2251        return sprintf(buf, "%d\n", test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
2252}
2253
2254void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
2255{
2256        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2257
2258        if (umcast_val > 0) {
2259                set_bit(IPOIB_FLAG_UMCAST, &priv->flags);
2260                ipoib_warn(priv, "ignoring multicast groups joined directly "
2261                                "by userspace\n");
2262        } else
2263                clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);
2264}
2265
2266static ssize_t set_umcast(struct device *dev,
2267                          struct device_attribute *attr,
2268                          const char *buf, size_t count)
2269{
2270        unsigned long umcast_val = simple_strtoul(buf, NULL, 0);
2271
2272        ipoib_set_umcast(to_net_dev(dev), umcast_val);
2273
2274        return count;
2275}
2276static DEVICE_ATTR(umcast, S_IWUSR | S_IRUGO, show_umcast, set_umcast);
2277
2278int ipoib_add_umcast_attr(struct net_device *dev)
2279{
2280        return device_create_file(&dev->dev, &dev_attr_umcast);
2281}
2282
2283static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
2284{
2285        struct ipoib_dev_priv *child_priv;
2286        struct net_device *netdev = priv->dev;
2287
2288        netif_addr_lock_bh(netdev);
2289
2290        memcpy(&priv->local_gid.global.interface_id,
2291               &gid->global.interface_id,
2292               sizeof(gid->global.interface_id));
2293        memcpy(netdev->dev_addr + 4, &priv->local_gid, sizeof(priv->local_gid));
2294        clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
2295
2296        netif_addr_unlock_bh(netdev);
2297
2298        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2299                down_read(&priv->vlan_rwsem);
2300                list_for_each_entry(child_priv, &priv->child_intfs, list)
2301                        set_base_guid(child_priv, gid);
2302                up_read(&priv->vlan_rwsem);
2303        }
2304}
2305
2306static int ipoib_check_lladdr(struct net_device *dev,
2307                              struct sockaddr_storage *ss)
2308{
2309        union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
2310        int ret = 0;
2311
2312        netif_addr_lock_bh(dev);
2313
2314        /* Make sure the QPN, reserved and subnet prefix match the current
2315         * lladdr, it also makes sure the lladdr is unicast.
2316         */
2317        if (memcmp(dev->dev_addr, ss->__data,
2318                   4 + sizeof(gid->global.subnet_prefix)) ||
2319            gid->global.interface_id == 0)
2320                ret = -EINVAL;
2321
2322        netif_addr_unlock_bh(dev);
2323
2324        return ret;
2325}
2326
2327static int ipoib_set_mac(struct net_device *dev, void *addr)
2328{
2329        struct ipoib_dev_priv *priv = ipoib_priv(dev);
2330        struct sockaddr_storage *ss = addr;
2331        int ret;
2332
2333        if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
2334                return -EBUSY;
2335
2336        ret = ipoib_check_lladdr(dev, ss);
2337        if (ret)
2338                return ret;
2339
2340        set_base_guid(priv, (union ib_gid *)(ss->__data + 4));
2341
2342        queue_work(ipoib_workqueue, &priv->flush_light);
2343
2344        return 0;
2345}
2346
2347static ssize_t create_child(struct device *dev,
2348                            struct device_attribute *attr,
2349                            const char *buf, size_t count)
2350{
2351        int pkey;
2352        int ret;
2353
2354        if (sscanf(buf, "%i", &pkey) != 1)
2355                return -EINVAL;
2356
2357        if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
2358                return -EINVAL;
2359
2360        ret = ipoib_vlan_add(to_net_dev(dev), pkey);
2361
2362        return ret ? ret : count;
2363}
2364static DEVICE_ATTR(create_child, S_IWUSR, NULL, create_child);
2365
2366static ssize_t delete_child(struct device *dev,
2367                            struct device_attribute *attr,
2368                            const char *buf, size_t count)
2369{
2370        int pkey;
2371        int ret;
2372
2373        if (sscanf(buf, "%i", &pkey) != 1)
2374                return -EINVAL;
2375
2376        if (pkey < 0 || pkey > 0xffff)
2377                return -EINVAL;
2378
2379        ret = ipoib_vlan_delete(to_net_dev(dev), pkey);
2380
2381        return ret ? ret : count;
2382
2383}
2384static DEVICE_ATTR(delete_child, S_IWUSR, NULL, delete_child);
2385
2386int ipoib_add_pkey_attr(struct net_device *dev)
2387{
2388        return device_create_file(&dev->dev, &dev_attr_pkey);
2389}
2390
2391/*
2392 * We erroneously exposed the iface's port number in the dev_id
2393 * sysfs field long after dev_port was introduced for that purpose[1],
2394 * and we need to stop everyone from relying on that.
2395 * Let's overload the shower routine for the dev_id file here
2396 * to gently bring the issue up.
2397 *
2398 * [1] https://www.spinics.net/lists/netdev/msg272123.html
2399 */
2400static ssize_t dev_id_show(struct device *dev,
2401                           struct device_attribute *attr, char *buf)
2402{
2403        struct net_device *ndev = to_net_dev(dev);
2404
2405        /*
2406         * ndev->dev_port will be equal to 0 in old kernel prior to commit
2407         * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
2408         * port numbers") Zero was chosen as special case for user space
2409         * applications to fallback and query dev_id to check if it has
2410         * different value or not.
2411         *
2412         * Don't print warning in such scenario.
2413         *
2414         * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
2415         */
2416        if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
2417                netdev_info_once(ndev,
2418                        "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
2419                        current->comm);
2420
2421        return sprintf(buf, "%#x\n", ndev->dev_id);
2422}
2423static DEVICE_ATTR_RO(dev_id);
2424
2425static int ipoib_intercept_dev_id_attr(struct net_device *dev)
2426{
2427        device_remove_file(&dev->dev, &dev_attr_dev_id);
2428        return device_create_file(&dev->dev, &dev_attr_dev_id);
2429}
2430
2431static struct net_device *ipoib_add_port(const char *format,
2432                                         struct ib_device *hca, u8 port)
2433{
2434        struct rtnl_link_ops *ops = ipoib_get_link_ops();
2435        struct rdma_netdev_alloc_params params;
2436        struct ipoib_dev_priv *priv;
2437        struct net_device *ndev;
2438        int result;
2439
2440        ndev = ipoib_intf_alloc(hca, port, format);
2441        if (IS_ERR(ndev)) {
2442                pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
2443                        PTR_ERR(ndev));
2444                return ndev;
2445        }
2446        priv = ipoib_priv(ndev);
2447
2448        INIT_IB_EVENT_HANDLER(&priv->event_handler,
2449                              priv->ca, ipoib_event);
2450        ib_register_event_handler(&priv->event_handler);
2451
2452        /* call event handler to ensure pkey in sync */
2453        queue_work(ipoib_workqueue, &priv->flush_heavy);
2454
2455        result = register_netdev(ndev);
2456        if (result) {
2457                pr_warn("%s: couldn't register ipoib port %d; error %d\n",
2458                        hca->name, port, result);
2459
2460                ipoib_parent_unregister_pre(ndev);
2461                ipoib_intf_free(ndev);
2462                free_netdev(ndev);
2463
2464                return ERR_PTR(result);
2465        }
2466
2467        if (hca->ops.rdma_netdev_get_params) {
2468                int rc = hca->ops.rdma_netdev_get_params(hca, port,
2469                                                     RDMA_NETDEV_IPOIB,
2470                                                     &params);
2471
2472                if (!rc && ops->priv_size < params.sizeof_priv)
2473                        ops->priv_size = params.sizeof_priv;
2474        }
2475        /*
2476         * We cannot set priv_destructor before register_netdev because we
2477         * need priv to be always valid during the error flow to execute
2478         * ipoib_parent_unregister_pre(). Instead handle it manually and only
2479         * enter priv_destructor mode once we are completely registered.
2480         */
2481        ndev->priv_destructor = ipoib_intf_free;
2482
2483        if (ipoib_intercept_dev_id_attr(ndev))
2484                goto sysfs_failed;
2485        if (ipoib_cm_add_mode_attr(ndev))
2486                goto sysfs_failed;
2487        if (ipoib_add_pkey_attr(ndev))
2488                goto sysfs_failed;
2489        if (ipoib_add_umcast_attr(ndev))
2490                goto sysfs_failed;
2491        if (device_create_file(&ndev->dev, &dev_attr_create_child))
2492                goto sysfs_failed;
2493        if (device_create_file(&ndev->dev, &dev_attr_delete_child))
2494                goto sysfs_failed;
2495
2496        return ndev;
2497
2498sysfs_failed:
2499        ipoib_parent_unregister_pre(ndev);
2500        unregister_netdev(ndev);
2501        return ERR_PTR(-ENOMEM);
2502}
2503
2504static void ipoib_add_one(struct ib_device *device)
2505{
2506        struct list_head *dev_list;
2507        struct net_device *dev;
2508        struct ipoib_dev_priv *priv;
2509        unsigned int p;
2510        int count = 0;
2511
2512        dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
2513        if (!dev_list)
2514                return;
2515
2516        INIT_LIST_HEAD(dev_list);
2517
2518        rdma_for_each_port (device, p) {
2519                if (!rdma_protocol_ib(device, p))
2520                        continue;
2521                dev = ipoib_add_port("ib%d", device, p);
2522                if (!IS_ERR(dev)) {
2523                        priv = ipoib_priv(dev);
2524                        list_add_tail(&priv->list, dev_list);
2525                        count++;
2526                }
2527        }
2528
2529        if (!count) {
2530                kfree(dev_list);
2531                return;
2532        }
2533
2534        ib_set_client_data(device, &ipoib_client, dev_list);
2535}
2536
2537static void ipoib_remove_one(struct ib_device *device, void *client_data)
2538{
2539        struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
2540        struct list_head *dev_list = client_data;
2541
2542        if (!dev_list)
2543                return;
2544
2545        list_for_each_entry_safe(priv, tmp, dev_list, list) {
2546                LIST_HEAD(head);
2547                ipoib_parent_unregister_pre(priv->dev);
2548
2549                rtnl_lock();
2550
2551                list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
2552                                         list)
2553                        unregister_netdevice_queue(cpriv->dev, &head);
2554                unregister_netdevice_queue(priv->dev, &head);
2555                unregister_netdevice_many(&head);
2556
2557                rtnl_unlock();
2558        }
2559
2560        kfree(dev_list);
2561}
2562
2563#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2564static struct notifier_block ipoib_netdev_notifier = {
2565        .notifier_call = ipoib_netdev_event,
2566};
2567#endif
2568
2569static int __init ipoib_init_module(void)
2570{
2571        int ret;
2572
2573        ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
2574        ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
2575        ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);
2576
2577        ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
2578        ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
2579        ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
2580#ifdef CONFIG_INFINIBAND_IPOIB_CM
2581        ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
2582        ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
2583#endif
2584
2585        /*
2586         * When copying small received packets, we only copy from the
2587         * linear data part of the SKB, so we rely on this condition.
2588         */
2589        BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);
2590
2591        ipoib_register_debugfs();
2592
2593        /*
2594         * We create a global workqueue here that is used for all flush
2595         * operations.  However, if you attempt to flush a workqueue
2596         * from a task on that same workqueue, it deadlocks the system.
2597         * We want to be able to flush the tasks associated with a
2598         * specific net device, so we also create a workqueue for each
2599         * netdevice.  We queue up the tasks for that device only on
2600         * its private workqueue, and we only queue up flush events
2601         * on our global flush workqueue.  This avoids the deadlocks.
2602         */
2603        ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
2604        if (!ipoib_workqueue) {
2605                ret = -ENOMEM;
2606                goto err_fs;
2607        }
2608
2609        ib_sa_register_client(&ipoib_sa_client);
2610
2611        ret = ib_register_client(&ipoib_client);
2612        if (ret)
2613                goto err_sa;
2614
2615        ret = ipoib_netlink_init();
2616        if (ret)
2617                goto err_client;
2618
2619#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2620        register_netdevice_notifier(&ipoib_netdev_notifier);
2621#endif
2622        return 0;
2623
2624err_client:
2625        ib_unregister_client(&ipoib_client);
2626
2627err_sa:
2628        ib_sa_unregister_client(&ipoib_sa_client);
2629        destroy_workqueue(ipoib_workqueue);
2630
2631err_fs:
2632        ipoib_unregister_debugfs();
2633
2634        return ret;
2635}
2636
2637static void __exit ipoib_cleanup_module(void)
2638{
2639#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2640        unregister_netdevice_notifier(&ipoib_netdev_notifier);
2641#endif
2642        ipoib_netlink_fini();
2643        ib_unregister_client(&ipoib_client);
2644        ib_sa_unregister_client(&ipoib_sa_client);
2645        ipoib_unregister_debugfs();
2646        destroy_workqueue(ipoib_workqueue);
2647}
2648
2649module_init(ipoib_init_module);
2650module_exit(ipoib_cleanup_module);
2651