linux/net/core/dev.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *      NET3    Protocol independent device support routines.
   4 *
   5 *      Derived from the non IP parts of dev.c 1.0.19
   6 *              Authors:        Ross Biro
   7 *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
   8 *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
   9 *
  10 *      Additional Authors:
  11 *              Florian la Roche <rzsfl@rz.uni-sb.de>
  12 *              Alan Cox <gw4pts@gw4pts.ampr.org>
  13 *              David Hinds <dahinds@users.sourceforge.net>
  14 *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  15 *              Adam Sulmicki <adam@cfar.umd.edu>
  16 *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  17 *
  18 *      Changes:
  19 *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  20 *                                      to 2 if register_netdev gets called
  21 *                                      before net_dev_init & also removed a
  22 *                                      few lines of code in the process.
  23 *              Alan Cox        :       device private ioctl copies fields back.
  24 *              Alan Cox        :       Transmit queue code does relevant
  25 *                                      stunts to keep the queue safe.
  26 *              Alan Cox        :       Fixed double lock.
  27 *              Alan Cox        :       Fixed promisc NULL pointer trap
  28 *              ????????        :       Support the full private ioctl range
  29 *              Alan Cox        :       Moved ioctl permission check into
  30 *                                      drivers
  31 *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  32 *              Alan Cox        :       100 backlog just doesn't cut it when
  33 *                                      you start doing multicast video 8)
  34 *              Alan Cox        :       Rewrote net_bh and list manager.
  35 *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  36 *              Alan Cox        :       Took out transmit every packet pass
  37 *                                      Saved a few bytes in the ioctl handler
  38 *              Alan Cox        :       Network driver sets packet type before
  39 *                                      calling netif_rx. Saves a function
  40 *                                      call a packet.
  41 *              Alan Cox        :       Hashed net_bh()
  42 *              Richard Kooijman:       Timestamp fixes.
  43 *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  44 *              Alan Cox        :       Device lock protection.
  45 *              Alan Cox        :       Fixed nasty side effect of device close
  46 *                                      changes.
  47 *              Rudi Cilibrasi  :       Pass the right thing to
  48 *                                      set_mac_address()
  49 *              Dave Miller     :       32bit quantity for the device lock to
  50 *                                      make it work out on a Sparc.
  51 *              Bjorn Ekwall    :       Added KERNELD hack.
  52 *              Alan Cox        :       Cleaned up the backlog initialise.
  53 *              Craig Metz      :       SIOCGIFCONF fix if space for under
  54 *                                      1 device.
  55 *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  56 *                                      is no device open function.
  57 *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  58 *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  59 *              Cyrus Durgin    :       Cleaned for KMOD
  60 *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  61 *                                      A network device unload needs to purge
  62 *                                      the backlog queue.
  63 *      Paul Rusty Russell      :       SIOCSIFNAME
  64 *              Pekka Riikonen  :       Netdev boot-time settings code
  65 *              Andrew Morton   :       Make unregister_netdevice wait
  66 *                                      indefinitely on dev->refcnt
  67 *              J Hadi Salim    :       - Backlog queue sampling
  68 *                                      - netif_rx() feedback
  69 */
  70
  71#include <linux/uaccess.h>
  72#include <linux/bitops.h>
  73#include <linux/capability.h>
  74#include <linux/cpu.h>
  75#include <linux/types.h>
  76#include <linux/kernel.h>
  77#include <linux/hash.h>
  78#include <linux/slab.h>
  79#include <linux/sched.h>
  80#include <linux/sched/mm.h>
  81#include <linux/mutex.h>
  82#include <linux/rwsem.h>
  83#include <linux/string.h>
  84#include <linux/mm.h>
  85#include <linux/socket.h>
  86#include <linux/sockios.h>
  87#include <linux/errno.h>
  88#include <linux/interrupt.h>
  89#include <linux/if_ether.h>
  90#include <linux/netdevice.h>
  91#include <linux/etherdevice.h>
  92#include <linux/ethtool.h>
  93#include <linux/skbuff.h>
  94#include <linux/kthread.h>
  95#include <linux/bpf.h>
  96#include <linux/bpf_trace.h>
  97#include <net/net_namespace.h>
  98#include <net/sock.h>
  99#include <net/busy_poll.h>
 100#include <linux/rtnetlink.h>
 101#include <linux/stat.h>
 102#include <net/dsa.h>
 103#include <net/dst.h>
 104#include <net/dst_metadata.h>
 105#include <net/gro.h>
 106#include <net/pkt_sched.h>
 107#include <net/pkt_cls.h>
 108#include <net/checksum.h>
 109#include <net/xfrm.h>
 110#include <linux/highmem.h>
 111#include <linux/init.h>
 112#include <linux/module.h>
 113#include <linux/netpoll.h>
 114#include <linux/rcupdate.h>
 115#include <linux/delay.h>
 116#include <net/iw_handler.h>
 117#include <asm/current.h>
 118#include <linux/audit.h>
 119#include <linux/dmaengine.h>
 120#include <linux/err.h>
 121#include <linux/ctype.h>
 122#include <linux/if_arp.h>
 123#include <linux/if_vlan.h>
 124#include <linux/ip.h>
 125#include <net/ip.h>
 126#include <net/mpls.h>
 127#include <linux/ipv6.h>
 128#include <linux/in.h>
 129#include <linux/jhash.h>
 130#include <linux/random.h>
 131#include <trace/events/napi.h>
 132#include <trace/events/net.h>
 133#include <trace/events/skb.h>
 134#include <trace/events/qdisc.h>
 135#include <linux/inetdevice.h>
 136#include <linux/cpu_rmap.h>
 137#include <linux/static_key.h>
 138#include <linux/hashtable.h>
 139#include <linux/vmalloc.h>
 140#include <linux/if_macvlan.h>
 141#include <linux/errqueue.h>
 142#include <linux/hrtimer.h>
 143#include <linux/netfilter_ingress.h>
 144#include <linux/crash_dump.h>
 145#include <linux/sctp.h>
 146#include <net/udp_tunnel.h>
 147#include <linux/net_namespace.h>
 148#include <linux/indirect_call_wrapper.h>
 149#include <net/devlink.h>
 150#include <linux/pm_runtime.h>
 151#include <linux/prandom.h>
 152#include <linux/once_lite.h>
 153
 154#include "net-sysfs.h"
 155
 156#define MAX_GRO_SKBS 8
 157
 158/* This should be increased if a protocol with a bigger head is added. */
 159#define GRO_MAX_HEAD (MAX_HEADER + 128)
 160
 161static DEFINE_SPINLOCK(ptype_lock);
 162static DEFINE_SPINLOCK(offload_lock);
 163struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 164struct list_head ptype_all __read_mostly;       /* Taps */
 165static struct list_head offload_base __read_mostly;
 166
 167static int netif_rx_internal(struct sk_buff *skb);
 168static int call_netdevice_notifiers_info(unsigned long val,
 169                                         struct netdev_notifier_info *info);
 170static int call_netdevice_notifiers_extack(unsigned long val,
 171                                           struct net_device *dev,
 172                                           struct netlink_ext_ack *extack);
 173static struct napi_struct *napi_by_id(unsigned int napi_id);
 174
 175/*
 176 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 177 * semaphore.
 178 *
 179 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 180 *
 181 * Writers must hold the rtnl semaphore while they loop through the
 182 * dev_base_head list, and hold dev_base_lock for writing when they do the
 183 * actual updates.  This allows pure readers to access the list even
 184 * while a writer is preparing to update it.
 185 *
 186 * To put it another way, dev_base_lock is held for writing only to
 187 * protect against pure readers; the rtnl semaphore provides the
 188 * protection against other writers.
 189 *
 190 * See, for example usages, register_netdevice() and
 191 * unregister_netdevice(), which must be called with the rtnl
 192 * semaphore held.
 193 */
 194DEFINE_RWLOCK(dev_base_lock);
 195EXPORT_SYMBOL(dev_base_lock);
 196
 197static DEFINE_MUTEX(ifalias_mutex);
 198
 199/* protects napi_hash addition/deletion and napi_gen_id */
 200static DEFINE_SPINLOCK(napi_hash_lock);
 201
 202static unsigned int napi_gen_id = NR_CPUS;
 203static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
 204
 205static DECLARE_RWSEM(devnet_rename_sem);
 206
 207static inline void dev_base_seq_inc(struct net *net)
 208{
 209        while (++net->dev_base_seq == 0)
 210                ;
 211}
 212
 213static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 214{
 215        unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
 216
 217        return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 218}
 219
 220static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 221{
 222        return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 223}
 224
 225static inline void rps_lock(struct softnet_data *sd)
 226{
 227#ifdef CONFIG_RPS
 228        spin_lock(&sd->input_pkt_queue.lock);
 229#endif
 230}
 231
 232static inline void rps_unlock(struct softnet_data *sd)
 233{
 234#ifdef CONFIG_RPS
 235        spin_unlock(&sd->input_pkt_queue.lock);
 236#endif
 237}
 238
 239static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
 240                                                       const char *name)
 241{
 242        struct netdev_name_node *name_node;
 243
 244        name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
 245        if (!name_node)
 246                return NULL;
 247        INIT_HLIST_NODE(&name_node->hlist);
 248        name_node->dev = dev;
 249        name_node->name = name;
 250        return name_node;
 251}
 252
 253static struct netdev_name_node *
 254netdev_name_node_head_alloc(struct net_device *dev)
 255{
 256        struct netdev_name_node *name_node;
 257
 258        name_node = netdev_name_node_alloc(dev, dev->name);
 259        if (!name_node)
 260                return NULL;
 261        INIT_LIST_HEAD(&name_node->list);
 262        return name_node;
 263}
 264
 265static void netdev_name_node_free(struct netdev_name_node *name_node)
 266{
 267        kfree(name_node);
 268}
 269
 270static void netdev_name_node_add(struct net *net,
 271                                 struct netdev_name_node *name_node)
 272{
 273        hlist_add_head_rcu(&name_node->hlist,
 274                           dev_name_hash(net, name_node->name));
 275}
 276
 277static void netdev_name_node_del(struct netdev_name_node *name_node)
 278{
 279        hlist_del_rcu(&name_node->hlist);
 280}
 281
 282static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
 283                                                        const char *name)
 284{
 285        struct hlist_head *head = dev_name_hash(net, name);
 286        struct netdev_name_node *name_node;
 287
 288        hlist_for_each_entry(name_node, head, hlist)
 289                if (!strcmp(name_node->name, name))
 290                        return name_node;
 291        return NULL;
 292}
 293
 294static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
 295                                                            const char *name)
 296{
 297        struct hlist_head *head = dev_name_hash(net, name);
 298        struct netdev_name_node *name_node;
 299
 300        hlist_for_each_entry_rcu(name_node, head, hlist)
 301                if (!strcmp(name_node->name, name))
 302                        return name_node;
 303        return NULL;
 304}
 305
 306int netdev_name_node_alt_create(struct net_device *dev, const char *name)
 307{
 308        struct netdev_name_node *name_node;
 309        struct net *net = dev_net(dev);
 310
 311        name_node = netdev_name_node_lookup(net, name);
 312        if (name_node)
 313                return -EEXIST;
 314        name_node = netdev_name_node_alloc(dev, name);
 315        if (!name_node)
 316                return -ENOMEM;
 317        netdev_name_node_add(net, name_node);
 318        /* The node that holds dev->name acts as a head of per-device list. */
 319        list_add_tail(&name_node->list, &dev->name_node->list);
 320
 321        return 0;
 322}
 323EXPORT_SYMBOL(netdev_name_node_alt_create);
 324
 325static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
 326{
 327        list_del(&name_node->list);
 328        netdev_name_node_del(name_node);
 329        kfree(name_node->name);
 330        netdev_name_node_free(name_node);
 331}
 332
 333int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
 334{
 335        struct netdev_name_node *name_node;
 336        struct net *net = dev_net(dev);
 337
 338        name_node = netdev_name_node_lookup(net, name);
 339        if (!name_node)
 340                return -ENOENT;
 341        /* lookup might have found our primary name or a name belonging
 342         * to another device.
 343         */
 344        if (name_node == dev->name_node || name_node->dev != dev)
 345                return -EINVAL;
 346
 347        __netdev_name_node_alt_destroy(name_node);
 348
 349        return 0;
 350}
 351EXPORT_SYMBOL(netdev_name_node_alt_destroy);
 352
 353static void netdev_name_node_alt_flush(struct net_device *dev)
 354{
 355        struct netdev_name_node *name_node, *tmp;
 356
 357        list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
 358                __netdev_name_node_alt_destroy(name_node);
 359}
 360
 361/* Device list insertion */
 362static void list_netdevice(struct net_device *dev)
 363{
 364        struct net *net = dev_net(dev);
 365
 366        ASSERT_RTNL();
 367
 368        write_lock_bh(&dev_base_lock);
 369        list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 370        netdev_name_node_add(net, dev->name_node);
 371        hlist_add_head_rcu(&dev->index_hlist,
 372                           dev_index_hash(net, dev->ifindex));
 373        write_unlock_bh(&dev_base_lock);
 374
 375        dev_base_seq_inc(net);
 376}
 377
 378/* Device list removal
 379 * caller must respect a RCU grace period before freeing/reusing dev
 380 */
 381static void unlist_netdevice(struct net_device *dev)
 382{
 383        ASSERT_RTNL();
 384
 385        /* Unlink dev from the device chain */
 386        write_lock_bh(&dev_base_lock);
 387        list_del_rcu(&dev->dev_list);
 388        netdev_name_node_del(dev->name_node);
 389        hlist_del_rcu(&dev->index_hlist);
 390        write_unlock_bh(&dev_base_lock);
 391
 392        dev_base_seq_inc(dev_net(dev));
 393}
 394
 395/*
 396 *      Our notifier list
 397 */
 398
 399static RAW_NOTIFIER_HEAD(netdev_chain);
 400
 401/*
 402 *      Device drivers call our routines to queue packets here. We empty the
 403 *      queue in the local softnet handler.
 404 */
 405
 406DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 407EXPORT_PER_CPU_SYMBOL(softnet_data);
 408
 409#ifdef CONFIG_LOCKDEP
 410/*
 411 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 412 * according to dev->type
 413 */
 414static const unsigned short netdev_lock_type[] = {
 415         ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 416         ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 417         ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 418         ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 419         ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 420         ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 421         ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 422         ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 423         ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 424         ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 425         ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 426         ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 427         ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
 428         ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
 429         ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
 430
 431static const char *const netdev_lock_name[] = {
 432        "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 433        "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 434        "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 435        "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 436        "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 437        "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 438        "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 439        "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 440        "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 441        "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 442        "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 443        "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 444        "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
 445        "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
 446        "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
 447
 448static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 449static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 450
 451static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 452{
 453        int i;
 454
 455        for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 456                if (netdev_lock_type[i] == dev_type)
 457                        return i;
 458        /* the last key is used by default */
 459        return ARRAY_SIZE(netdev_lock_type) - 1;
 460}
 461
 462static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 463                                                 unsigned short dev_type)
 464{
 465        int i;
 466
 467        i = netdev_lock_pos(dev_type);
 468        lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 469                                   netdev_lock_name[i]);
 470}
 471
 472static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 473{
 474        int i;
 475
 476        i = netdev_lock_pos(dev->type);
 477        lockdep_set_class_and_name(&dev->addr_list_lock,
 478                                   &netdev_addr_lock_key[i],
 479                                   netdev_lock_name[i]);
 480}
 481#else
 482static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 483                                                 unsigned short dev_type)
 484{
 485}
 486
 487static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 488{
 489}
 490#endif
 491
 492/*******************************************************************************
 493 *
 494 *              Protocol management and registration routines
 495 *
 496 *******************************************************************************/
 497
 498
 499/*
 500 *      Add a protocol ID to the list. Now that the input handler is
 501 *      smarter we can dispense with all the messy stuff that used to be
 502 *      here.
 503 *
 504 *      BEWARE!!! Protocol handlers, mangling input packets,
 505 *      MUST BE last in hash buckets and checking protocol handlers
 506 *      MUST start from promiscuous ptype_all chain in net_bh.
 507 *      It is true now, do not change it.
 508 *      Explanation follows: if protocol handler, mangling packet, will
 509 *      be the first on list, it is not able to sense, that packet
 510 *      is cloned and should be copied-on-write, so that it will
 511 *      change it and subsequent readers will get broken packet.
 512 *                                                      --ANK (980803)
 513 */
 514
 515static inline struct list_head *ptype_head(const struct packet_type *pt)
 516{
 517        if (pt->type == htons(ETH_P_ALL))
 518                return pt->dev ? &pt->dev->ptype_all : &ptype_all;
 519        else
 520                return pt->dev ? &pt->dev->ptype_specific :
 521                                 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 522}
 523
 524/**
 525 *      dev_add_pack - add packet handler
 526 *      @pt: packet type declaration
 527 *
 528 *      Add a protocol handler to the networking stack. The passed &packet_type
 529 *      is linked into kernel lists and may not be freed until it has been
 530 *      removed from the kernel lists.
 531 *
 532 *      This call does not sleep therefore it can not
 533 *      guarantee all CPU's that are in middle of receiving packets
 534 *      will see the new packet type (until the next received packet).
 535 */
 536
 537void dev_add_pack(struct packet_type *pt)
 538{
 539        struct list_head *head = ptype_head(pt);
 540
 541        spin_lock(&ptype_lock);
 542        list_add_rcu(&pt->list, head);
 543        spin_unlock(&ptype_lock);
 544}
 545EXPORT_SYMBOL(dev_add_pack);
 546
 547/**
 548 *      __dev_remove_pack        - remove packet handler
 549 *      @pt: packet type declaration
 550 *
 551 *      Remove a protocol handler that was previously added to the kernel
 552 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 553 *      from the kernel lists and can be freed or reused once this function
 554 *      returns.
 555 *
 556 *      The packet type might still be in use by receivers
 557 *      and must not be freed until after all the CPU's have gone
 558 *      through a quiescent state.
 559 */
 560void __dev_remove_pack(struct packet_type *pt)
 561{
 562        struct list_head *head = ptype_head(pt);
 563        struct packet_type *pt1;
 564
 565        spin_lock(&ptype_lock);
 566
 567        list_for_each_entry(pt1, head, list) {
 568                if (pt == pt1) {
 569                        list_del_rcu(&pt->list);
 570                        goto out;
 571                }
 572        }
 573
 574        pr_warn("dev_remove_pack: %p not found\n", pt);
 575out:
 576        spin_unlock(&ptype_lock);
 577}
 578EXPORT_SYMBOL(__dev_remove_pack);
 579
 580/**
 581 *      dev_remove_pack  - remove packet handler
 582 *      @pt: packet type declaration
 583 *
 584 *      Remove a protocol handler that was previously added to the kernel
 585 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 586 *      from the kernel lists and can be freed or reused once this function
 587 *      returns.
 588 *
 589 *      This call sleeps to guarantee that no CPU is looking at the packet
 590 *      type after return.
 591 */
 592void dev_remove_pack(struct packet_type *pt)
 593{
 594        __dev_remove_pack(pt);
 595
 596        synchronize_net();
 597}
 598EXPORT_SYMBOL(dev_remove_pack);
 599
 600
 601/**
 602 *      dev_add_offload - register offload handlers
 603 *      @po: protocol offload declaration
 604 *
 605 *      Add protocol offload handlers to the networking stack. The passed
 606 *      &proto_offload is linked into kernel lists and may not be freed until
 607 *      it has been removed from the kernel lists.
 608 *
 609 *      This call does not sleep therefore it can not
 610 *      guarantee all CPU's that are in middle of receiving packets
 611 *      will see the new offload handlers (until the next received packet).
 612 */
 613void dev_add_offload(struct packet_offload *po)
 614{
 615        struct packet_offload *elem;
 616
 617        spin_lock(&offload_lock);
 618        list_for_each_entry(elem, &offload_base, list) {
 619                if (po->priority < elem->priority)
 620                        break;
 621        }
 622        list_add_rcu(&po->list, elem->list.prev);
 623        spin_unlock(&offload_lock);
 624}
 625EXPORT_SYMBOL(dev_add_offload);
 626
 627/**
 628 *      __dev_remove_offload     - remove offload handler
 629 *      @po: packet offload declaration
 630 *
 631 *      Remove a protocol offload handler that was previously added to the
 632 *      kernel offload handlers by dev_add_offload(). The passed &offload_type
 633 *      is removed from the kernel lists and can be freed or reused once this
 634 *      function returns.
 635 *
 636 *      The packet type might still be in use by receivers
 637 *      and must not be freed until after all the CPU's have gone
 638 *      through a quiescent state.
 639 */
 640static void __dev_remove_offload(struct packet_offload *po)
 641{
 642        struct list_head *head = &offload_base;
 643        struct packet_offload *po1;
 644
 645        spin_lock(&offload_lock);
 646
 647        list_for_each_entry(po1, head, list) {
 648                if (po == po1) {
 649                        list_del_rcu(&po->list);
 650                        goto out;
 651                }
 652        }
 653
 654        pr_warn("dev_remove_offload: %p not found\n", po);
 655out:
 656        spin_unlock(&offload_lock);
 657}
 658
 659/**
 660 *      dev_remove_offload       - remove packet offload handler
 661 *      @po: packet offload declaration
 662 *
 663 *      Remove a packet offload handler that was previously added to the kernel
 664 *      offload handlers by dev_add_offload(). The passed &offload_type is
 665 *      removed from the kernel lists and can be freed or reused once this
 666 *      function returns.
 667 *
 668 *      This call sleeps to guarantee that no CPU is looking at the packet
 669 *      type after return.
 670 */
 671void dev_remove_offload(struct packet_offload *po)
 672{
 673        __dev_remove_offload(po);
 674
 675        synchronize_net();
 676}
 677EXPORT_SYMBOL(dev_remove_offload);
 678
 679/*******************************************************************************
 680 *
 681 *                          Device Interface Subroutines
 682 *
 683 *******************************************************************************/
 684
 685/**
 686 *      dev_get_iflink  - get 'iflink' value of a interface
 687 *      @dev: targeted interface
 688 *
 689 *      Indicates the ifindex the interface is linked to.
 690 *      Physical interfaces have the same 'ifindex' and 'iflink' values.
 691 */
 692
 693int dev_get_iflink(const struct net_device *dev)
 694{
 695        if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
 696                return dev->netdev_ops->ndo_get_iflink(dev);
 697
 698        return dev->ifindex;
 699}
 700EXPORT_SYMBOL(dev_get_iflink);
 701
 702/**
 703 *      dev_fill_metadata_dst - Retrieve tunnel egress information.
 704 *      @dev: targeted interface
 705 *      @skb: The packet.
 706 *
 707 *      For better visibility of tunnel traffic OVS needs to retrieve
 708 *      egress tunnel information for a packet. Following API allows
 709 *      user to get this info.
 710 */
 711int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 712{
 713        struct ip_tunnel_info *info;
 714
 715        if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
 716                return -EINVAL;
 717
 718        info = skb_tunnel_info_unclone(skb);
 719        if (!info)
 720                return -ENOMEM;
 721        if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
 722                return -EINVAL;
 723
 724        return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
 725}
 726EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
 727
 728static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
 729{
 730        int k = stack->num_paths++;
 731
 732        if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
 733                return NULL;
 734
 735        return &stack->path[k];
 736}
 737
 738int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
 739                          struct net_device_path_stack *stack)
 740{
 741        const struct net_device *last_dev;
 742        struct net_device_path_ctx ctx = {
 743                .dev    = dev,
 744                .daddr  = daddr,
 745        };
 746        struct net_device_path *path;
 747        int ret = 0;
 748
 749        stack->num_paths = 0;
 750        while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
 751                last_dev = ctx.dev;
 752                path = dev_fwd_path(stack);
 753                if (!path)
 754                        return -1;
 755
 756                memset(path, 0, sizeof(struct net_device_path));
 757                ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
 758                if (ret < 0)
 759                        return -1;
 760
 761                if (WARN_ON_ONCE(last_dev == ctx.dev))
 762                        return -1;
 763        }
 764        path = dev_fwd_path(stack);
 765        if (!path)
 766                return -1;
 767        path->type = DEV_PATH_ETHERNET;
 768        path->dev = ctx.dev;
 769
 770        return ret;
 771}
 772EXPORT_SYMBOL_GPL(dev_fill_forward_path);
 773
 774/**
 775 *      __dev_get_by_name       - find a device by its name
 776 *      @net: the applicable net namespace
 777 *      @name: name to find
 778 *
 779 *      Find an interface by name. Must be called under RTNL semaphore
 780 *      or @dev_base_lock. If the name is found a pointer to the device
 781 *      is returned. If the name is not found then %NULL is returned. The
 782 *      reference counters are not incremented so the caller must be
 783 *      careful with locks.
 784 */
 785
 786struct net_device *__dev_get_by_name(struct net *net, const char *name)
 787{
 788        struct netdev_name_node *node_name;
 789
 790        node_name = netdev_name_node_lookup(net, name);
 791        return node_name ? node_name->dev : NULL;
 792}
 793EXPORT_SYMBOL(__dev_get_by_name);
 794
 795/**
 796 * dev_get_by_name_rcu  - find a device by its name
 797 * @net: the applicable net namespace
 798 * @name: name to find
 799 *
 800 * Find an interface by name.
 801 * If the name is found a pointer to the device is returned.
 802 * If the name is not found then %NULL is returned.
 803 * The reference counters are not incremented so the caller must be
 804 * careful with locks. The caller must hold RCU lock.
 805 */
 806
 807struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 808{
 809        struct netdev_name_node *node_name;
 810
 811        node_name = netdev_name_node_lookup_rcu(net, name);
 812        return node_name ? node_name->dev : NULL;
 813}
 814EXPORT_SYMBOL(dev_get_by_name_rcu);
 815
 816/**
 817 *      dev_get_by_name         - find a device by its name
 818 *      @net: the applicable net namespace
 819 *      @name: name to find
 820 *
 821 *      Find an interface by name. This can be called from any
 822 *      context and does its own locking. The returned handle has
 823 *      the usage count incremented and the caller must use dev_put() to
 824 *      release it when it is no longer needed. %NULL is returned if no
 825 *      matching device is found.
 826 */
 827
 828struct net_device *dev_get_by_name(struct net *net, const char *name)
 829{
 830        struct net_device *dev;
 831
 832        rcu_read_lock();
 833        dev = dev_get_by_name_rcu(net, name);
 834        dev_hold(dev);
 835        rcu_read_unlock();
 836        return dev;
 837}
 838EXPORT_SYMBOL(dev_get_by_name);
 839
 840/**
 841 *      __dev_get_by_index - find a device by its ifindex
 842 *      @net: the applicable net namespace
 843 *      @ifindex: index of device
 844 *
 845 *      Search for an interface by index. Returns %NULL if the device
 846 *      is not found or a pointer to the device. The device has not
 847 *      had its reference counter increased so the caller must be careful
 848 *      about locking. The caller must hold either the RTNL semaphore
 849 *      or @dev_base_lock.
 850 */
 851
 852struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 853{
 854        struct net_device *dev;
 855        struct hlist_head *head = dev_index_hash(net, ifindex);
 856
 857        hlist_for_each_entry(dev, head, index_hlist)
 858                if (dev->ifindex == ifindex)
 859                        return dev;
 860
 861        return NULL;
 862}
 863EXPORT_SYMBOL(__dev_get_by_index);
 864
 865/**
 866 *      dev_get_by_index_rcu - find a device by its ifindex
 867 *      @net: the applicable net namespace
 868 *      @ifindex: index of device
 869 *
 870 *      Search for an interface by index. Returns %NULL if the device
 871 *      is not found or a pointer to the device. The device has not
 872 *      had its reference counter increased so the caller must be careful
 873 *      about locking. The caller must hold RCU lock.
 874 */
 875
 876struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 877{
 878        struct net_device *dev;
 879        struct hlist_head *head = dev_index_hash(net, ifindex);
 880
 881        hlist_for_each_entry_rcu(dev, head, index_hlist)
 882                if (dev->ifindex == ifindex)
 883                        return dev;
 884
 885        return NULL;
 886}
 887EXPORT_SYMBOL(dev_get_by_index_rcu);
 888
 889
 890/**
 891 *      dev_get_by_index - find a device by its ifindex
 892 *      @net: the applicable net namespace
 893 *      @ifindex: index of device
 894 *
 895 *      Search for an interface by index. Returns NULL if the device
 896 *      is not found or a pointer to the device. The device returned has
 897 *      had a reference added and the pointer is safe until the user calls
 898 *      dev_put to indicate they have finished with it.
 899 */
 900
 901struct net_device *dev_get_by_index(struct net *net, int ifindex)
 902{
 903        struct net_device *dev;
 904
 905        rcu_read_lock();
 906        dev = dev_get_by_index_rcu(net, ifindex);
 907        dev_hold(dev);
 908        rcu_read_unlock();
 909        return dev;
 910}
 911EXPORT_SYMBOL(dev_get_by_index);
 912
 913/**
 914 *      dev_get_by_napi_id - find a device by napi_id
 915 *      @napi_id: ID of the NAPI struct
 916 *
 917 *      Search for an interface by NAPI ID. Returns %NULL if the device
 918 *      is not found or a pointer to the device. The device has not had
 919 *      its reference counter increased so the caller must be careful
 920 *      about locking. The caller must hold RCU lock.
 921 */
 922
 923struct net_device *dev_get_by_napi_id(unsigned int napi_id)
 924{
 925        struct napi_struct *napi;
 926
 927        WARN_ON_ONCE(!rcu_read_lock_held());
 928
 929        if (napi_id < MIN_NAPI_ID)
 930                return NULL;
 931
 932        napi = napi_by_id(napi_id);
 933
 934        return napi ? napi->dev : NULL;
 935}
 936EXPORT_SYMBOL(dev_get_by_napi_id);
 937
 938/**
 939 *      netdev_get_name - get a netdevice name, knowing its ifindex.
 940 *      @net: network namespace
 941 *      @name: a pointer to the buffer where the name will be stored.
 942 *      @ifindex: the ifindex of the interface to get the name from.
 943 */
 944int netdev_get_name(struct net *net, char *name, int ifindex)
 945{
 946        struct net_device *dev;
 947        int ret;
 948
 949        down_read(&devnet_rename_sem);
 950        rcu_read_lock();
 951
 952        dev = dev_get_by_index_rcu(net, ifindex);
 953        if (!dev) {
 954                ret = -ENODEV;
 955                goto out;
 956        }
 957
 958        strcpy(name, dev->name);
 959
 960        ret = 0;
 961out:
 962        rcu_read_unlock();
 963        up_read(&devnet_rename_sem);
 964        return ret;
 965}
 966
 967/**
 968 *      dev_getbyhwaddr_rcu - find a device by its hardware address
 969 *      @net: the applicable net namespace
 970 *      @type: media type of device
 971 *      @ha: hardware address
 972 *
 973 *      Search for an interface by MAC address. Returns NULL if the device
 974 *      is not found or a pointer to the device.
 975 *      The caller must hold RCU or RTNL.
 976 *      The returned device has not had its ref count increased
 977 *      and the caller must therefore be careful about locking
 978 *
 979 */
 980
 981struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
 982                                       const char *ha)
 983{
 984        struct net_device *dev;
 985
 986        for_each_netdev_rcu(net, dev)
 987                if (dev->type == type &&
 988                    !memcmp(dev->dev_addr, ha, dev->addr_len))
 989                        return dev;
 990
 991        return NULL;
 992}
 993EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
 994
 995struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 996{
 997        struct net_device *dev, *ret = NULL;
 998
 999        rcu_read_lock();
1000        for_each_netdev_rcu(net, dev)
1001                if (dev->type == type) {
1002                        dev_hold(dev);
1003                        ret = dev;
1004                        break;
1005                }
1006        rcu_read_unlock();
1007        return ret;
1008}
1009EXPORT_SYMBOL(dev_getfirstbyhwtype);
1010
1011/**
1012 *      __dev_get_by_flags - find any device with given flags
1013 *      @net: the applicable net namespace
1014 *      @if_flags: IFF_* values
1015 *      @mask: bitmask of bits in if_flags to check
1016 *
1017 *      Search for any interface with the given flags. Returns NULL if a device
1018 *      is not found or a pointer to the device. Must be called inside
1019 *      rtnl_lock(), and result refcount is unchanged.
1020 */
1021
1022struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1023                                      unsigned short mask)
1024{
1025        struct net_device *dev, *ret;
1026
1027        ASSERT_RTNL();
1028
1029        ret = NULL;
1030        for_each_netdev(net, dev) {
1031                if (((dev->flags ^ if_flags) & mask) == 0) {
1032                        ret = dev;
1033                        break;
1034                }
1035        }
1036        return ret;
1037}
1038EXPORT_SYMBOL(__dev_get_by_flags);
1039
1040/**
1041 *      dev_valid_name - check if name is okay for network device
1042 *      @name: name string
1043 *
1044 *      Network device names need to be valid file names to
1045 *      allow sysfs to work.  We also disallow any kind of
1046 *      whitespace.
1047 */
1048bool dev_valid_name(const char *name)
1049{
1050        if (*name == '\0')
1051                return false;
1052        if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1053                return false;
1054        if (!strcmp(name, ".") || !strcmp(name, ".."))
1055                return false;
1056
1057        while (*name) {
1058                if (*name == '/' || *name == ':' || isspace(*name))
1059                        return false;
1060                name++;
1061        }
1062        return true;
1063}
1064EXPORT_SYMBOL(dev_valid_name);
1065
1066/**
1067 *      __dev_alloc_name - allocate a name for a device
1068 *      @net: network namespace to allocate the device name in
1069 *      @name: name format string
1070 *      @buf:  scratch buffer and result name string
1071 *
1072 *      Passed a format string - eg "lt%d" it will try and find a suitable
1073 *      id. It scans list of devices to build up a free map, then chooses
1074 *      the first empty slot. The caller must hold the dev_base or rtnl lock
1075 *      while allocating the name and adding the device in order to avoid
1076 *      duplicates.
1077 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1078 *      Returns the number of the unit assigned or a negative errno code.
1079 */
1080
1081static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1082{
1083        int i = 0;
1084        const char *p;
1085        const int max_netdevices = 8*PAGE_SIZE;
1086        unsigned long *inuse;
1087        struct net_device *d;
1088
1089        if (!dev_valid_name(name))
1090                return -EINVAL;
1091
1092        p = strchr(name, '%');
1093        if (p) {
1094                /*
1095                 * Verify the string as this thing may have come from
1096                 * the user.  There must be either one "%d" and no other "%"
1097                 * characters.
1098                 */
1099                if (p[1] != 'd' || strchr(p + 2, '%'))
1100                        return -EINVAL;
1101
1102                /* Use one page as a bit array of possible slots */
1103                inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1104                if (!inuse)
1105                        return -ENOMEM;
1106
1107                for_each_netdev(net, d) {
1108                        struct netdev_name_node *name_node;
1109                        list_for_each_entry(name_node, &d->name_node->list, list) {
1110                                if (!sscanf(name_node->name, name, &i))
1111                                        continue;
1112                                if (i < 0 || i >= max_netdevices)
1113                                        continue;
1114
1115                                /*  avoid cases where sscanf is not exact inverse of printf */
1116                                snprintf(buf, IFNAMSIZ, name, i);
1117                                if (!strncmp(buf, name_node->name, IFNAMSIZ))
1118                                        set_bit(i, inuse);
1119                        }
1120                        if (!sscanf(d->name, name, &i))
1121                                continue;
1122                        if (i < 0 || i >= max_netdevices)
1123                                continue;
1124
1125                        /*  avoid cases where sscanf is not exact inverse of printf */
1126                        snprintf(buf, IFNAMSIZ, name, i);
1127                        if (!strncmp(buf, d->name, IFNAMSIZ))
1128                                set_bit(i, inuse);
1129                }
1130
1131                i = find_first_zero_bit(inuse, max_netdevices);
1132                free_page((unsigned long) inuse);
1133        }
1134
1135        snprintf(buf, IFNAMSIZ, name, i);
1136        if (!__dev_get_by_name(net, buf))
1137                return i;
1138
1139        /* It is possible to run out of possible slots
1140         * when the name is long and there isn't enough space left
1141         * for the digits, or if all bits are used.
1142         */
1143        return -ENFILE;
1144}
1145
1146static int dev_alloc_name_ns(struct net *net,
1147                             struct net_device *dev,
1148                             const char *name)
1149{
1150        char buf[IFNAMSIZ];
1151        int ret;
1152
1153        BUG_ON(!net);
1154        ret = __dev_alloc_name(net, name, buf);
1155        if (ret >= 0)
1156                strlcpy(dev->name, buf, IFNAMSIZ);
1157        return ret;
1158}
1159
1160/**
1161 *      dev_alloc_name - allocate a name for a device
1162 *      @dev: device
1163 *      @name: name format string
1164 *
1165 *      Passed a format string - eg "lt%d" it will try and find a suitable
1166 *      id. It scans list of devices to build up a free map, then chooses
1167 *      the first empty slot. The caller must hold the dev_base or rtnl lock
1168 *      while allocating the name and adding the device in order to avoid
1169 *      duplicates.
1170 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1171 *      Returns the number of the unit assigned or a negative errno code.
1172 */
1173
1174int dev_alloc_name(struct net_device *dev, const char *name)
1175{
1176        return dev_alloc_name_ns(dev_net(dev), dev, name);
1177}
1178EXPORT_SYMBOL(dev_alloc_name);
1179
1180static int dev_get_valid_name(struct net *net, struct net_device *dev,
1181                              const char *name)
1182{
1183        BUG_ON(!net);
1184
1185        if (!dev_valid_name(name))
1186                return -EINVAL;
1187
1188        if (strchr(name, '%'))
1189                return dev_alloc_name_ns(net, dev, name);
1190        else if (__dev_get_by_name(net, name))
1191                return -EEXIST;
1192        else if (dev->name != name)
1193                strlcpy(dev->name, name, IFNAMSIZ);
1194
1195        return 0;
1196}
1197
1198/**
1199 *      dev_change_name - change name of a device
1200 *      @dev: device
1201 *      @newname: name (or format string) must be at least IFNAMSIZ
1202 *
1203 *      Change name of a device, can pass format strings "eth%d".
1204 *      for wildcarding.
1205 */
1206int dev_change_name(struct net_device *dev, const char *newname)
1207{
1208        unsigned char old_assign_type;
1209        char oldname[IFNAMSIZ];
1210        int err = 0;
1211        int ret;
1212        struct net *net;
1213
1214        ASSERT_RTNL();
1215        BUG_ON(!dev_net(dev));
1216
1217        net = dev_net(dev);
1218
1219        /* Some auto-enslaved devices e.g. failover slaves are
1220         * special, as userspace might rename the device after
1221         * the interface had been brought up and running since
1222         * the point kernel initiated auto-enslavement. Allow
1223         * live name change even when these slave devices are
1224         * up and running.
1225         *
1226         * Typically, users of these auto-enslaving devices
1227         * don't actually care about slave name change, as
1228         * they are supposed to operate on master interface
1229         * directly.
1230         */
1231        if (dev->flags & IFF_UP &&
1232            likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1233                return -EBUSY;
1234
1235        down_write(&devnet_rename_sem);
1236
1237        if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1238                up_write(&devnet_rename_sem);
1239                return 0;
1240        }
1241
1242        memcpy(oldname, dev->name, IFNAMSIZ);
1243
1244        err = dev_get_valid_name(net, dev, newname);
1245        if (err < 0) {
1246                up_write(&devnet_rename_sem);
1247                return err;
1248        }
1249
1250        if (oldname[0] && !strchr(oldname, '%'))
1251                netdev_info(dev, "renamed from %s\n", oldname);
1252
1253        old_assign_type = dev->name_assign_type;
1254        dev->name_assign_type = NET_NAME_RENAMED;
1255
1256rollback:
1257        ret = device_rename(&dev->dev, dev->name);
1258        if (ret) {
1259                memcpy(dev->name, oldname, IFNAMSIZ);
1260                dev->name_assign_type = old_assign_type;
1261                up_write(&devnet_rename_sem);
1262                return ret;
1263        }
1264
1265        up_write(&devnet_rename_sem);
1266
1267        netdev_adjacent_rename_links(dev, oldname);
1268
1269        write_lock_bh(&dev_base_lock);
1270        netdev_name_node_del(dev->name_node);
1271        write_unlock_bh(&dev_base_lock);
1272
1273        synchronize_rcu();
1274
1275        write_lock_bh(&dev_base_lock);
1276        netdev_name_node_add(net, dev->name_node);
1277        write_unlock_bh(&dev_base_lock);
1278
1279        ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1280        ret = notifier_to_errno(ret);
1281
1282        if (ret) {
1283                /* err >= 0 after dev_alloc_name() or stores the first errno */
1284                if (err >= 0) {
1285                        err = ret;
1286                        down_write(&devnet_rename_sem);
1287                        memcpy(dev->name, oldname, IFNAMSIZ);
1288                        memcpy(oldname, newname, IFNAMSIZ);
1289                        dev->name_assign_type = old_assign_type;
1290                        old_assign_type = NET_NAME_RENAMED;
1291                        goto rollback;
1292                } else {
1293                        pr_err("%s: name change rollback failed: %d\n",
1294                               dev->name, ret);
1295                }
1296        }
1297
1298        return err;
1299}
1300
1301/**
1302 *      dev_set_alias - change ifalias of a device
1303 *      @dev: device
1304 *      @alias: name up to IFALIASZ
1305 *      @len: limit of bytes to copy from info
1306 *
1307 *      Set ifalias for a device,
1308 */
1309int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1310{
1311        struct dev_ifalias *new_alias = NULL;
1312
1313        if (len >= IFALIASZ)
1314                return -EINVAL;
1315
1316        if (len) {
1317                new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1318                if (!new_alias)
1319                        return -ENOMEM;
1320
1321                memcpy(new_alias->ifalias, alias, len);
1322                new_alias->ifalias[len] = 0;
1323        }
1324
1325        mutex_lock(&ifalias_mutex);
1326        new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1327                                        mutex_is_locked(&ifalias_mutex));
1328        mutex_unlock(&ifalias_mutex);
1329
1330        if (new_alias)
1331                kfree_rcu(new_alias, rcuhead);
1332
1333        return len;
1334}
1335EXPORT_SYMBOL(dev_set_alias);
1336
1337/**
1338 *      dev_get_alias - get ifalias of a device
1339 *      @dev: device
1340 *      @name: buffer to store name of ifalias
1341 *      @len: size of buffer
1342 *
1343 *      get ifalias for a device.  Caller must make sure dev cannot go
1344 *      away,  e.g. rcu read lock or own a reference count to device.
1345 */
1346int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1347{
1348        const struct dev_ifalias *alias;
1349        int ret = 0;
1350
1351        rcu_read_lock();
1352        alias = rcu_dereference(dev->ifalias);
1353        if (alias)
1354                ret = snprintf(name, len, "%s", alias->ifalias);
1355        rcu_read_unlock();
1356
1357        return ret;
1358}
1359
1360/**
1361 *      netdev_features_change - device changes features
1362 *      @dev: device to cause notification
1363 *
1364 *      Called to indicate a device has changed features.
1365 */
1366void netdev_features_change(struct net_device *dev)
1367{
1368        call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1369}
1370EXPORT_SYMBOL(netdev_features_change);
1371
1372/**
1373 *      netdev_state_change - device changes state
1374 *      @dev: device to cause notification
1375 *
1376 *      Called to indicate a device has changed state. This function calls
1377 *      the notifier chains for netdev_chain and sends a NEWLINK message
1378 *      to the routing socket.
1379 */
1380void netdev_state_change(struct net_device *dev)
1381{
1382        if (dev->flags & IFF_UP) {
1383                struct netdev_notifier_change_info change_info = {
1384                        .info.dev = dev,
1385                };
1386
1387                call_netdevice_notifiers_info(NETDEV_CHANGE,
1388                                              &change_info.info);
1389                rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1390        }
1391}
1392EXPORT_SYMBOL(netdev_state_change);
1393
1394/**
1395 * __netdev_notify_peers - notify network peers about existence of @dev,
1396 * to be called when rtnl lock is already held.
1397 * @dev: network device
1398 *
1399 * Generate traffic such that interested network peers are aware of
1400 * @dev, such as by generating a gratuitous ARP. This may be used when
1401 * a device wants to inform the rest of the network about some sort of
1402 * reconfiguration such as a failover event or virtual machine
1403 * migration.
1404 */
1405void __netdev_notify_peers(struct net_device *dev)
1406{
1407        ASSERT_RTNL();
1408        call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1409        call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1410}
1411EXPORT_SYMBOL(__netdev_notify_peers);
1412
1413/**
1414 * netdev_notify_peers - notify network peers about existence of @dev
1415 * @dev: network device
1416 *
1417 * Generate traffic such that interested network peers are aware of
1418 * @dev, such as by generating a gratuitous ARP. This may be used when
1419 * a device wants to inform the rest of the network about some sort of
1420 * reconfiguration such as a failover event or virtual machine
1421 * migration.
1422 */
1423void netdev_notify_peers(struct net_device *dev)
1424{
1425        rtnl_lock();
1426        __netdev_notify_peers(dev);
1427        rtnl_unlock();
1428}
1429EXPORT_SYMBOL(netdev_notify_peers);
1430
1431static int napi_threaded_poll(void *data);
1432
1433static int napi_kthread_create(struct napi_struct *n)
1434{
1435        int err = 0;
1436
1437        /* Create and wake up the kthread once to put it in
1438         * TASK_INTERRUPTIBLE mode to avoid the blocked task
1439         * warning and work with loadavg.
1440         */
1441        n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1442                                n->dev->name, n->napi_id);
1443        if (IS_ERR(n->thread)) {
1444                err = PTR_ERR(n->thread);
1445                pr_err("kthread_run failed with err %d\n", err);
1446                n->thread = NULL;
1447        }
1448
1449        return err;
1450}
1451
1452static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1453{
1454        const struct net_device_ops *ops = dev->netdev_ops;
1455        int ret;
1456
1457        ASSERT_RTNL();
1458
1459        if (!netif_device_present(dev)) {
1460                /* may be detached because parent is runtime-suspended */
1461                if (dev->dev.parent)
1462                        pm_runtime_resume(dev->dev.parent);
1463                if (!netif_device_present(dev))
1464                        return -ENODEV;
1465        }
1466
1467        /* Block netpoll from trying to do any rx path servicing.
1468         * If we don't do this there is a chance ndo_poll_controller
1469         * or ndo_poll may be running while we open the device
1470         */
1471        netpoll_poll_disable(dev);
1472
1473        ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1474        ret = notifier_to_errno(ret);
1475        if (ret)
1476                return ret;
1477
1478        set_bit(__LINK_STATE_START, &dev->state);
1479
1480        if (ops->ndo_validate_addr)
1481                ret = ops->ndo_validate_addr(dev);
1482
1483        if (!ret && ops->ndo_open)
1484                ret = ops->ndo_open(dev);
1485
1486        netpoll_poll_enable(dev);
1487
1488        if (ret)
1489                clear_bit(__LINK_STATE_START, &dev->state);
1490        else {
1491                dev->flags |= IFF_UP;
1492                dev_set_rx_mode(dev);
1493                dev_activate(dev);
1494                add_device_randomness(dev->dev_addr, dev->addr_len);
1495        }
1496
1497        return ret;
1498}
1499
1500/**
1501 *      dev_open        - prepare an interface for use.
1502 *      @dev: device to open
1503 *      @extack: netlink extended ack
1504 *
1505 *      Takes a device from down to up state. The device's private open
1506 *      function is invoked and then the multicast lists are loaded. Finally
1507 *      the device is moved into the up state and a %NETDEV_UP message is
1508 *      sent to the netdev notifier chain.
1509 *
1510 *      Calling this function on an active interface is a nop. On a failure
1511 *      a negative errno code is returned.
1512 */
1513int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1514{
1515        int ret;
1516
1517        if (dev->flags & IFF_UP)
1518                return 0;
1519
1520        ret = __dev_open(dev, extack);
1521        if (ret < 0)
1522                return ret;
1523
1524        rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1525        call_netdevice_notifiers(NETDEV_UP, dev);
1526
1527        return ret;
1528}
1529EXPORT_SYMBOL(dev_open);
1530
1531static void __dev_close_many(struct list_head *head)
1532{
1533        struct net_device *dev;
1534
1535        ASSERT_RTNL();
1536        might_sleep();
1537
1538        list_for_each_entry(dev, head, close_list) {
1539                /* Temporarily disable netpoll until the interface is down */
1540                netpoll_poll_disable(dev);
1541
1542                call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1543
1544                clear_bit(__LINK_STATE_START, &dev->state);
1545
1546                /* Synchronize to scheduled poll. We cannot touch poll list, it
1547                 * can be even on different cpu. So just clear netif_running().
1548                 *
1549                 * dev->stop() will invoke napi_disable() on all of it's
1550                 * napi_struct instances on this device.
1551                 */
1552                smp_mb__after_atomic(); /* Commit netif_running(). */
1553        }
1554
1555        dev_deactivate_many(head);
1556
1557        list_for_each_entry(dev, head, close_list) {
1558                const struct net_device_ops *ops = dev->netdev_ops;
1559
1560                /*
1561                 *      Call the device specific close. This cannot fail.
1562                 *      Only if device is UP
1563                 *
1564                 *      We allow it to be called even after a DETACH hot-plug
1565                 *      event.
1566                 */
1567                if (ops->ndo_stop)
1568                        ops->ndo_stop(dev);
1569
1570                dev->flags &= ~IFF_UP;
1571                netpoll_poll_enable(dev);
1572        }
1573}
1574
1575static void __dev_close(struct net_device *dev)
1576{
1577        LIST_HEAD(single);
1578
1579        list_add(&dev->close_list, &single);
1580        __dev_close_many(&single);
1581        list_del(&single);
1582}
1583
1584void dev_close_many(struct list_head *head, bool unlink)
1585{
1586        struct net_device *dev, *tmp;
1587
1588        /* Remove the devices that don't need to be closed */
1589        list_for_each_entry_safe(dev, tmp, head, close_list)
1590                if (!(dev->flags & IFF_UP))
1591                        list_del_init(&dev->close_list);
1592
1593        __dev_close_many(head);
1594
1595        list_for_each_entry_safe(dev, tmp, head, close_list) {
1596                rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1597                call_netdevice_notifiers(NETDEV_DOWN, dev);
1598                if (unlink)
1599                        list_del_init(&dev->close_list);
1600        }
1601}
1602EXPORT_SYMBOL(dev_close_many);
1603
1604/**
1605 *      dev_close - shutdown an interface.
1606 *      @dev: device to shutdown
1607 *
1608 *      This function moves an active device into down state. A
1609 *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1610 *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1611 *      chain.
1612 */
1613void dev_close(struct net_device *dev)
1614{
1615        if (dev->flags & IFF_UP) {
1616                LIST_HEAD(single);
1617
1618                list_add(&dev->close_list, &single);
1619                dev_close_many(&single, true);
1620                list_del(&single);
1621        }
1622}
1623EXPORT_SYMBOL(dev_close);
1624
1625
1626/**
1627 *      dev_disable_lro - disable Large Receive Offload on a device
1628 *      @dev: device
1629 *
1630 *      Disable Large Receive Offload (LRO) on a net device.  Must be
1631 *      called under RTNL.  This is needed if received packets may be
1632 *      forwarded to another interface.
1633 */
1634void dev_disable_lro(struct net_device *dev)
1635{
1636        struct net_device *lower_dev;
1637        struct list_head *iter;
1638
1639        dev->wanted_features &= ~NETIF_F_LRO;
1640        netdev_update_features(dev);
1641
1642        if (unlikely(dev->features & NETIF_F_LRO))
1643                netdev_WARN(dev, "failed to disable LRO!\n");
1644
1645        netdev_for_each_lower_dev(dev, lower_dev, iter)
1646                dev_disable_lro(lower_dev);
1647}
1648EXPORT_SYMBOL(dev_disable_lro);
1649
1650/**
1651 *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1652 *      @dev: device
1653 *
1654 *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1655 *      called under RTNL.  This is needed if Generic XDP is installed on
1656 *      the device.
1657 */
1658static void dev_disable_gro_hw(struct net_device *dev)
1659{
1660        dev->wanted_features &= ~NETIF_F_GRO_HW;
1661        netdev_update_features(dev);
1662
1663        if (unlikely(dev->features & NETIF_F_GRO_HW))
1664                netdev_WARN(dev, "failed to disable GRO_HW!\n");
1665}
1666
1667const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1668{
1669#define N(val)                                          \
1670        case NETDEV_##val:                              \
1671                return "NETDEV_" __stringify(val);
1672        switch (cmd) {
1673        N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1674        N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1675        N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1676        N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1677        N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1678        N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1679        N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1680        N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1681        N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1682        N(PRE_CHANGEADDR)
1683        }
1684#undef N
1685        return "UNKNOWN_NETDEV_EVENT";
1686}
1687EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1688
1689static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1690                                   struct net_device *dev)
1691{
1692        struct netdev_notifier_info info = {
1693                .dev = dev,
1694        };
1695
1696        return nb->notifier_call(nb, val, &info);
1697}
1698
1699static int call_netdevice_register_notifiers(struct notifier_block *nb,
1700                                             struct net_device *dev)
1701{
1702        int err;
1703
1704        err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1705        err = notifier_to_errno(err);
1706        if (err)
1707                return err;
1708
1709        if (!(dev->flags & IFF_UP))
1710                return 0;
1711
1712        call_netdevice_notifier(nb, NETDEV_UP, dev);
1713        return 0;
1714}
1715
1716static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1717                                                struct net_device *dev)
1718{
1719        if (dev->flags & IFF_UP) {
1720                call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1721                                        dev);
1722                call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1723        }
1724        call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1725}
1726
1727static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1728                                                 struct net *net)
1729{
1730        struct net_device *dev;
1731        int err;
1732
1733        for_each_netdev(net, dev) {
1734                err = call_netdevice_register_notifiers(nb, dev);
1735                if (err)
1736                        goto rollback;
1737        }
1738        return 0;
1739
1740rollback:
1741        for_each_netdev_continue_reverse(net, dev)
1742                call_netdevice_unregister_notifiers(nb, dev);
1743        return err;
1744}
1745
1746static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1747                                                    struct net *net)
1748{
1749        struct net_device *dev;
1750
1751        for_each_netdev(net, dev)
1752                call_netdevice_unregister_notifiers(nb, dev);
1753}
1754
1755static int dev_boot_phase = 1;
1756
1757/**
1758 * register_netdevice_notifier - register a network notifier block
1759 * @nb: notifier
1760 *
1761 * Register a notifier to be called when network device events occur.
1762 * The notifier passed is linked into the kernel structures and must
1763 * not be reused until it has been unregistered. A negative errno code
1764 * is returned on a failure.
1765 *
1766 * When registered all registration and up events are replayed
1767 * to the new notifier to allow device to have a race free
1768 * view of the network device list.
1769 */
1770
1771int register_netdevice_notifier(struct notifier_block *nb)
1772{
1773        struct net *net;
1774        int err;
1775
1776        /* Close race with setup_net() and cleanup_net() */
1777        down_write(&pernet_ops_rwsem);
1778        rtnl_lock();
1779        err = raw_notifier_chain_register(&netdev_chain, nb);
1780        if (err)
1781                goto unlock;
1782        if (dev_boot_phase)
1783                goto unlock;
1784        for_each_net(net) {
1785                err = call_netdevice_register_net_notifiers(nb, net);
1786                if (err)
1787                        goto rollback;
1788        }
1789
1790unlock:
1791        rtnl_unlock();
1792        up_write(&pernet_ops_rwsem);
1793        return err;
1794
1795rollback:
1796        for_each_net_continue_reverse(net)
1797                call_netdevice_unregister_net_notifiers(nb, net);
1798
1799        raw_notifier_chain_unregister(&netdev_chain, nb);
1800        goto unlock;
1801}
1802EXPORT_SYMBOL(register_netdevice_notifier);
1803
1804/**
1805 * unregister_netdevice_notifier - unregister a network notifier block
1806 * @nb: notifier
1807 *
1808 * Unregister a notifier previously registered by
1809 * register_netdevice_notifier(). The notifier is unlinked into the
1810 * kernel structures and may then be reused. A negative errno code
1811 * is returned on a failure.
1812 *
1813 * After unregistering unregister and down device events are synthesized
1814 * for all devices on the device list to the removed notifier to remove
1815 * the need for special case cleanup code.
1816 */
1817
1818int unregister_netdevice_notifier(struct notifier_block *nb)
1819{
1820        struct net *net;
1821        int err;
1822
1823        /* Close race with setup_net() and cleanup_net() */
1824        down_write(&pernet_ops_rwsem);
1825        rtnl_lock();
1826        err = raw_notifier_chain_unregister(&netdev_chain, nb);
1827        if (err)
1828                goto unlock;
1829
1830        for_each_net(net)
1831                call_netdevice_unregister_net_notifiers(nb, net);
1832
1833unlock:
1834        rtnl_unlock();
1835        up_write(&pernet_ops_rwsem);
1836        return err;
1837}
1838EXPORT_SYMBOL(unregister_netdevice_notifier);
1839
1840static int __register_netdevice_notifier_net(struct net *net,
1841                                             struct notifier_block *nb,
1842                                             bool ignore_call_fail)
1843{
1844        int err;
1845
1846        err = raw_notifier_chain_register(&net->netdev_chain, nb);
1847        if (err)
1848                return err;
1849        if (dev_boot_phase)
1850                return 0;
1851
1852        err = call_netdevice_register_net_notifiers(nb, net);
1853        if (err && !ignore_call_fail)
1854                goto chain_unregister;
1855
1856        return 0;
1857
1858chain_unregister:
1859        raw_notifier_chain_unregister(&net->netdev_chain, nb);
1860        return err;
1861}
1862
1863static int __unregister_netdevice_notifier_net(struct net *net,
1864                                               struct notifier_block *nb)
1865{
1866        int err;
1867
1868        err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1869        if (err)
1870                return err;
1871
1872        call_netdevice_unregister_net_notifiers(nb, net);
1873        return 0;
1874}
1875
1876/**
1877 * register_netdevice_notifier_net - register a per-netns network notifier block
1878 * @net: network namespace
1879 * @nb: notifier
1880 *
1881 * Register a notifier to be called when network device events occur.
1882 * The notifier passed is linked into the kernel structures and must
1883 * not be reused until it has been unregistered. A negative errno code
1884 * is returned on a failure.
1885 *
1886 * When registered all registration and up events are replayed
1887 * to the new notifier to allow device to have a race free
1888 * view of the network device list.
1889 */
1890
1891int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1892{
1893        int err;
1894
1895        rtnl_lock();
1896        err = __register_netdevice_notifier_net(net, nb, false);
1897        rtnl_unlock();
1898        return err;
1899}
1900EXPORT_SYMBOL(register_netdevice_notifier_net);
1901
1902/**
1903 * unregister_netdevice_notifier_net - unregister a per-netns
1904 *                                     network notifier block
1905 * @net: network namespace
1906 * @nb: notifier
1907 *
1908 * Unregister a notifier previously registered by
1909 * register_netdevice_notifier(). The notifier is unlinked into the
1910 * kernel structures and may then be reused. A negative errno code
1911 * is returned on a failure.
1912 *
1913 * After unregistering unregister and down device events are synthesized
1914 * for all devices on the device list to the removed notifier to remove
1915 * the need for special case cleanup code.
1916 */
1917
1918int unregister_netdevice_notifier_net(struct net *net,
1919                                      struct notifier_block *nb)
1920{
1921        int err;
1922
1923        rtnl_lock();
1924        err = __unregister_netdevice_notifier_net(net, nb);
1925        rtnl_unlock();
1926        return err;
1927}
1928EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1929
1930int register_netdevice_notifier_dev_net(struct net_device *dev,
1931                                        struct notifier_block *nb,
1932                                        struct netdev_net_notifier *nn)
1933{
1934        int err;
1935
1936        rtnl_lock();
1937        err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1938        if (!err) {
1939                nn->nb = nb;
1940                list_add(&nn->list, &dev->net_notifier_list);
1941        }
1942        rtnl_unlock();
1943        return err;
1944}
1945EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1946
1947int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1948                                          struct notifier_block *nb,
1949                                          struct netdev_net_notifier *nn)
1950{
1951        int err;
1952
1953        rtnl_lock();
1954        list_del(&nn->list);
1955        err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1956        rtnl_unlock();
1957        return err;
1958}
1959EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1960
1961static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1962                                             struct net *net)
1963{
1964        struct netdev_net_notifier *nn;
1965
1966        list_for_each_entry(nn, &dev->net_notifier_list, list) {
1967                __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
1968                __register_netdevice_notifier_net(net, nn->nb, true);
1969        }
1970}
1971
1972/**
1973 *      call_netdevice_notifiers_info - call all network notifier blocks
1974 *      @val: value passed unmodified to notifier function
1975 *      @info: notifier information data
1976 *
1977 *      Call all network notifier blocks.  Parameters and return value
1978 *      are as for raw_notifier_call_chain().
1979 */
1980
1981static int call_netdevice_notifiers_info(unsigned long val,
1982                                         struct netdev_notifier_info *info)
1983{
1984        struct net *net = dev_net(info->dev);
1985        int ret;
1986
1987        ASSERT_RTNL();
1988
1989        /* Run per-netns notifier block chain first, then run the global one.
1990         * Hopefully, one day, the global one is going to be removed after
1991         * all notifier block registrators get converted to be per-netns.
1992         */
1993        ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1994        if (ret & NOTIFY_STOP_MASK)
1995                return ret;
1996        return raw_notifier_call_chain(&netdev_chain, val, info);
1997}
1998
1999static int call_netdevice_notifiers_extack(unsigned long val,
2000                                           struct net_device *dev,
2001                                           struct netlink_ext_ack *extack)
2002{
2003        struct netdev_notifier_info info = {
2004                .dev = dev,
2005                .extack = extack,
2006        };
2007
2008        return call_netdevice_notifiers_info(val, &info);
2009}
2010
2011/**
2012 *      call_netdevice_notifiers - call all network notifier blocks
2013 *      @val: value passed unmodified to notifier function
2014 *      @dev: net_device pointer passed unmodified to notifier function
2015 *
2016 *      Call all network notifier blocks.  Parameters and return value
2017 *      are as for raw_notifier_call_chain().
2018 */
2019
2020int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2021{
2022        return call_netdevice_notifiers_extack(val, dev, NULL);
2023}
2024EXPORT_SYMBOL(call_netdevice_notifiers);
2025
2026/**
2027 *      call_netdevice_notifiers_mtu - call all network notifier blocks
2028 *      @val: value passed unmodified to notifier function
2029 *      @dev: net_device pointer passed unmodified to notifier function
2030 *      @arg: additional u32 argument passed to the notifier function
2031 *
2032 *      Call all network notifier blocks.  Parameters and return value
2033 *      are as for raw_notifier_call_chain().
2034 */
2035static int call_netdevice_notifiers_mtu(unsigned long val,
2036                                        struct net_device *dev, u32 arg)
2037{
2038        struct netdev_notifier_info_ext info = {
2039                .info.dev = dev,
2040                .ext.mtu = arg,
2041        };
2042
2043        BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2044
2045        return call_netdevice_notifiers_info(val, &info.info);
2046}
2047
2048#ifdef CONFIG_NET_INGRESS
2049static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2050
2051void net_inc_ingress_queue(void)
2052{
2053        static_branch_inc(&ingress_needed_key);
2054}
2055EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2056
2057void net_dec_ingress_queue(void)
2058{
2059        static_branch_dec(&ingress_needed_key);
2060}
2061EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2062#endif
2063
2064#ifdef CONFIG_NET_EGRESS
2065static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2066
2067void net_inc_egress_queue(void)
2068{
2069        static_branch_inc(&egress_needed_key);
2070}
2071EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2072
2073void net_dec_egress_queue(void)
2074{
2075        static_branch_dec(&egress_needed_key);
2076}
2077EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2078#endif
2079
2080static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2081#ifdef CONFIG_JUMP_LABEL
2082static atomic_t netstamp_needed_deferred;
2083static atomic_t netstamp_wanted;
2084static void netstamp_clear(struct work_struct *work)
2085{
2086        int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2087        int wanted;
2088
2089        wanted = atomic_add_return(deferred, &netstamp_wanted);
2090        if (wanted > 0)
2091                static_branch_enable(&netstamp_needed_key);
2092        else
2093                static_branch_disable(&netstamp_needed_key);
2094}
2095static DECLARE_WORK(netstamp_work, netstamp_clear);
2096#endif
2097
2098void net_enable_timestamp(void)
2099{
2100#ifdef CONFIG_JUMP_LABEL
2101        int wanted;
2102
2103        while (1) {
2104                wanted = atomic_read(&netstamp_wanted);
2105                if (wanted <= 0)
2106                        break;
2107                if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2108                        return;
2109        }
2110        atomic_inc(&netstamp_needed_deferred);
2111        schedule_work(&netstamp_work);
2112#else
2113        static_branch_inc(&netstamp_needed_key);
2114#endif
2115}
2116EXPORT_SYMBOL(net_enable_timestamp);
2117
2118void net_disable_timestamp(void)
2119{
2120#ifdef CONFIG_JUMP_LABEL
2121        int wanted;
2122
2123        while (1) {
2124                wanted = atomic_read(&netstamp_wanted);
2125                if (wanted <= 1)
2126                        break;
2127                if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2128                        return;
2129        }
2130        atomic_dec(&netstamp_needed_deferred);
2131        schedule_work(&netstamp_work);
2132#else
2133        static_branch_dec(&netstamp_needed_key);
2134#endif
2135}
2136EXPORT_SYMBOL(net_disable_timestamp);
2137
2138static inline void net_timestamp_set(struct sk_buff *skb)
2139{
2140        skb->tstamp = 0;
2141        if (static_branch_unlikely(&netstamp_needed_key))
2142                __net_timestamp(skb);
2143}
2144
2145#define net_timestamp_check(COND, SKB)                          \
2146        if (static_branch_unlikely(&netstamp_needed_key)) {     \
2147                if ((COND) && !(SKB)->tstamp)                   \
2148                        __net_timestamp(SKB);                   \
2149        }                                                       \
2150
2151bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2152{
2153        return __is_skb_forwardable(dev, skb, true);
2154}
2155EXPORT_SYMBOL_GPL(is_skb_forwardable);
2156
2157static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2158                              bool check_mtu)
2159{
2160        int ret = ____dev_forward_skb(dev, skb, check_mtu);
2161
2162        if (likely(!ret)) {
2163                skb->protocol = eth_type_trans(skb, dev);
2164                skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2165        }
2166
2167        return ret;
2168}
2169
2170int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2171{
2172        return __dev_forward_skb2(dev, skb, true);
2173}
2174EXPORT_SYMBOL_GPL(__dev_forward_skb);
2175
2176/**
2177 * dev_forward_skb - loopback an skb to another netif
2178 *
2179 * @dev: destination network device
2180 * @skb: buffer to forward
2181 *
2182 * return values:
2183 *      NET_RX_SUCCESS  (no congestion)
2184 *      NET_RX_DROP     (packet was dropped, but freed)
2185 *
2186 * dev_forward_skb can be used for injecting an skb from the
2187 * start_xmit function of one device into the receive queue
2188 * of another device.
2189 *
2190 * The receiving device may be in another namespace, so
2191 * we have to clear all information in the skb that could
2192 * impact namespace isolation.
2193 */
2194int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2195{
2196        return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2197}
2198EXPORT_SYMBOL_GPL(dev_forward_skb);
2199
2200int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2201{
2202        return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2203}
2204
2205static inline int deliver_skb(struct sk_buff *skb,
2206                              struct packet_type *pt_prev,
2207                              struct net_device *orig_dev)
2208{
2209        if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2210                return -ENOMEM;
2211        refcount_inc(&skb->users);
2212        return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2213}
2214
2215static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2216                                          struct packet_type **pt,
2217                                          struct net_device *orig_dev,
2218                                          __be16 type,
2219                                          struct list_head *ptype_list)
2220{
2221        struct packet_type *ptype, *pt_prev = *pt;
2222
2223        list_for_each_entry_rcu(ptype, ptype_list, list) {
2224                if (ptype->type != type)
2225                        continue;
2226                if (pt_prev)
2227                        deliver_skb(skb, pt_prev, orig_dev);
2228                pt_prev = ptype;
2229        }
2230        *pt = pt_prev;
2231}
2232
2233static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2234{
2235        if (!ptype->af_packet_priv || !skb->sk)
2236                return false;
2237
2238        if (ptype->id_match)
2239                return ptype->id_match(ptype, skb->sk);
2240        else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2241                return true;
2242
2243        return false;
2244}
2245
2246/**
2247 * dev_nit_active - return true if any network interface taps are in use
2248 *
2249 * @dev: network device to check for the presence of taps
2250 */
2251bool dev_nit_active(struct net_device *dev)
2252{
2253        return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2254}
2255EXPORT_SYMBOL_GPL(dev_nit_active);
2256
2257/*
2258 *      Support routine. Sends outgoing frames to any network
2259 *      taps currently in use.
2260 */
2261
2262void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2263{
2264        struct packet_type *ptype;
2265        struct sk_buff *skb2 = NULL;
2266        struct packet_type *pt_prev = NULL;
2267        struct list_head *ptype_list = &ptype_all;
2268
2269        rcu_read_lock();
2270again:
2271        list_for_each_entry_rcu(ptype, ptype_list, list) {
2272                if (ptype->ignore_outgoing)
2273                        continue;
2274
2275                /* Never send packets back to the socket
2276                 * they originated from - MvS (miquels@drinkel.ow.org)
2277                 */
2278                if (skb_loop_sk(ptype, skb))
2279                        continue;
2280
2281                if (pt_prev) {
2282                        deliver_skb(skb2, pt_prev, skb->dev);
2283                        pt_prev = ptype;
2284                        continue;
2285                }
2286
2287                /* need to clone skb, done only once */
2288                skb2 = skb_clone(skb, GFP_ATOMIC);
2289                if (!skb2)
2290                        goto out_unlock;
2291
2292                net_timestamp_set(skb2);
2293
2294                /* skb->nh should be correctly
2295                 * set by sender, so that the second statement is
2296                 * just protection against buggy protocols.
2297                 */
2298                skb_reset_mac_header(skb2);
2299
2300                if (skb_network_header(skb2) < skb2->data ||
2301                    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2302                        net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2303                                             ntohs(skb2->protocol),
2304                                             dev->name);
2305                        skb_reset_network_header(skb2);
2306                }
2307
2308                skb2->transport_header = skb2->network_header;
2309                skb2->pkt_type = PACKET_OUTGOING;
2310                pt_prev = ptype;
2311        }
2312
2313        if (ptype_list == &ptype_all) {
2314                ptype_list = &dev->ptype_all;
2315                goto again;
2316        }
2317out_unlock:
2318        if (pt_prev) {
2319                if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2320                        pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2321                else
2322                        kfree_skb(skb2);
2323        }
2324        rcu_read_unlock();
2325}
2326EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2327
2328/**
2329 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2330 * @dev: Network device
2331 * @txq: number of queues available
2332 *
2333 * If real_num_tx_queues is changed the tc mappings may no longer be
2334 * valid. To resolve this verify the tc mapping remains valid and if
2335 * not NULL the mapping. With no priorities mapping to this
2336 * offset/count pair it will no longer be used. In the worst case TC0
2337 * is invalid nothing can be done so disable priority mappings. If is
2338 * expected that drivers will fix this mapping if they can before
2339 * calling netif_set_real_num_tx_queues.
2340 */
2341static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2342{
2343        int i;
2344        struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2345
2346        /* If TC0 is invalidated disable TC mapping */
2347        if (tc->offset + tc->count > txq) {
2348                pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2349                dev->num_tc = 0;
2350                return;
2351        }
2352
2353        /* Invalidated prio to tc mappings set to TC0 */
2354        for (i = 1; i < TC_BITMASK + 1; i++) {
2355                int q = netdev_get_prio_tc_map(dev, i);
2356
2357                tc = &dev->tc_to_txq[q];
2358                if (tc->offset + tc->count > txq) {
2359                        pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2360                                i, q);
2361                        netdev_set_prio_tc_map(dev, i, 0);
2362                }
2363        }
2364}
2365
2366int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2367{
2368        if (dev->num_tc) {
2369                struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2370                int i;
2371
2372                /* walk through the TCs and see if it falls into any of them */
2373                for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2374                        if ((txq - tc->offset) < tc->count)
2375                                return i;
2376                }
2377
2378                /* didn't find it, just return -1 to indicate no match */
2379                return -1;
2380        }
2381
2382        return 0;
2383}
2384EXPORT_SYMBOL(netdev_txq_to_tc);
2385
2386#ifdef CONFIG_XPS
2387static struct static_key xps_needed __read_mostly;
2388static struct static_key xps_rxqs_needed __read_mostly;
2389static DEFINE_MUTEX(xps_map_mutex);
2390#define xmap_dereference(P)             \
2391        rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2392
2393static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2394                             struct xps_dev_maps *old_maps, int tci, u16 index)
2395{
2396        struct xps_map *map = NULL;
2397        int pos;
2398
2399        if (dev_maps)
2400                map = xmap_dereference(dev_maps->attr_map[tci]);
2401        if (!map)
2402                return false;
2403
2404        for (pos = map->len; pos--;) {
2405                if (map->queues[pos] != index)
2406                        continue;
2407
2408                if (map->len > 1) {
2409                        map->queues[pos] = map->queues[--map->len];
2410                        break;
2411                }
2412
2413                if (old_maps)
2414                        RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2415                RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2416                kfree_rcu(map, rcu);
2417                return false;
2418        }
2419
2420        return true;
2421}
2422
2423static bool remove_xps_queue_cpu(struct net_device *dev,
2424                                 struct xps_dev_maps *dev_maps,
2425                                 int cpu, u16 offset, u16 count)
2426{
2427        int num_tc = dev_maps->num_tc;
2428        bool active = false;
2429        int tci;
2430
2431        for (tci = cpu * num_tc; num_tc--; tci++) {
2432                int i, j;
2433
2434                for (i = count, j = offset; i--; j++) {
2435                        if (!remove_xps_queue(dev_maps, NULL, tci, j))
2436                                break;
2437                }
2438
2439                active |= i < 0;
2440        }
2441
2442        return active;
2443}
2444
2445static void reset_xps_maps(struct net_device *dev,
2446                           struct xps_dev_maps *dev_maps,
2447                           enum xps_map_type type)
2448{
2449        static_key_slow_dec_cpuslocked(&xps_needed);
2450        if (type == XPS_RXQS)
2451                static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2452
2453        RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2454
2455        kfree_rcu(dev_maps, rcu);
2456}
2457
2458static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2459                           u16 offset, u16 count)
2460{
2461        struct xps_dev_maps *dev_maps;
2462        bool active = false;
2463        int i, j;
2464
2465        dev_maps = xmap_dereference(dev->xps_maps[type]);
2466        if (!dev_maps)
2467                return;
2468
2469        for (j = 0; j < dev_maps->nr_ids; j++)
2470                active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2471        if (!active)
2472                reset_xps_maps(dev, dev_maps, type);
2473
2474        if (type == XPS_CPUS) {
2475                for (i = offset + (count - 1); count--; i--)
2476                        netdev_queue_numa_node_write(
2477                                netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2478        }
2479}
2480
2481static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2482                                   u16 count)
2483{
2484        if (!static_key_false(&xps_needed))
2485                return;
2486
2487        cpus_read_lock();
2488        mutex_lock(&xps_map_mutex);
2489
2490        if (static_key_false(&xps_rxqs_needed))
2491                clean_xps_maps(dev, XPS_RXQS, offset, count);
2492
2493        clean_xps_maps(dev, XPS_CPUS, offset, count);
2494
2495        mutex_unlock(&xps_map_mutex);
2496        cpus_read_unlock();
2497}
2498
2499static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2500{
2501        netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2502}
2503
2504static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2505                                      u16 index, bool is_rxqs_map)
2506{
2507        struct xps_map *new_map;
2508        int alloc_len = XPS_MIN_MAP_ALLOC;
2509        int i, pos;
2510
2511        for (pos = 0; map && pos < map->len; pos++) {
2512                if (map->queues[pos] != index)
2513                        continue;
2514                return map;
2515        }
2516
2517        /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2518        if (map) {
2519                if (pos < map->alloc_len)
2520                        return map;
2521
2522                alloc_len = map->alloc_len * 2;
2523        }
2524
2525        /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2526         *  map
2527         */
2528        if (is_rxqs_map)
2529                new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2530        else
2531                new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2532                                       cpu_to_node(attr_index));
2533        if (!new_map)
2534                return NULL;
2535
2536        for (i = 0; i < pos; i++)
2537                new_map->queues[i] = map->queues[i];
2538        new_map->alloc_len = alloc_len;
2539        new_map->len = pos;
2540
2541        return new_map;
2542}
2543
2544/* Copy xps maps at a given index */
2545static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2546                              struct xps_dev_maps *new_dev_maps, int index,
2547                              int tc, bool skip_tc)
2548{
2549        int i, tci = index * dev_maps->num_tc;
2550        struct xps_map *map;
2551
2552        /* copy maps belonging to foreign traffic classes */
2553        for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2554                if (i == tc && skip_tc)
2555                        continue;
2556
2557                /* fill in the new device map from the old device map */
2558                map = xmap_dereference(dev_maps->attr_map[tci]);
2559                RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2560        }
2561}
2562
2563/* Must be called under cpus_read_lock */
2564int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2565                          u16 index, enum xps_map_type type)
2566{
2567        struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2568        const unsigned long *online_mask = NULL;
2569        bool active = false, copy = false;
2570        int i, j, tci, numa_node_id = -2;
2571        int maps_sz, num_tc = 1, tc = 0;
2572        struct xps_map *map, *new_map;
2573        unsigned int nr_ids;
2574
2575        if (dev->num_tc) {
2576                /* Do not allow XPS on subordinate device directly */
2577                num_tc = dev->num_tc;
2578                if (num_tc < 0)
2579                        return -EINVAL;
2580
2581                /* If queue belongs to subordinate dev use its map */
2582                dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2583
2584                tc = netdev_txq_to_tc(dev, index);
2585                if (tc < 0)
2586                        return -EINVAL;
2587        }
2588
2589        mutex_lock(&xps_map_mutex);
2590
2591        dev_maps = xmap_dereference(dev->xps_maps[type]);
2592        if (type == XPS_RXQS) {
2593                maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2594                nr_ids = dev->num_rx_queues;
2595        } else {
2596                maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2597                if (num_possible_cpus() > 1)
2598                        online_mask = cpumask_bits(cpu_online_mask);
2599                nr_ids = nr_cpu_ids;
2600        }
2601
2602        if (maps_sz < L1_CACHE_BYTES)
2603                maps_sz = L1_CACHE_BYTES;
2604
2605        /* The old dev_maps could be larger or smaller than the one we're
2606         * setting up now, as dev->num_tc or nr_ids could have been updated in
2607         * between. We could try to be smart, but let's be safe instead and only
2608         * copy foreign traffic classes if the two map sizes match.
2609         */
2610        if (dev_maps &&
2611            dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2612                copy = true;
2613
2614        /* allocate memory for queue storage */
2615        for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2616             j < nr_ids;) {
2617                if (!new_dev_maps) {
2618                        new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2619                        if (!new_dev_maps) {
2620                                mutex_unlock(&xps_map_mutex);
2621                                return -ENOMEM;
2622                        }
2623
2624                        new_dev_maps->nr_ids = nr_ids;
2625                        new_dev_maps->num_tc = num_tc;
2626                }
2627
2628                tci = j * num_tc + tc;
2629                map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2630
2631                map = expand_xps_map(map, j, index, type == XPS_RXQS);
2632                if (!map)
2633                        goto error;
2634
2635                RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2636        }
2637
2638        if (!new_dev_maps)
2639                goto out_no_new_maps;
2640
2641        if (!dev_maps) {
2642                /* Increment static keys at most once per type */
2643                static_key_slow_inc_cpuslocked(&xps_needed);
2644                if (type == XPS_RXQS)
2645                        static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2646        }
2647
2648        for (j = 0; j < nr_ids; j++) {
2649                bool skip_tc = false;
2650
2651                tci = j * num_tc + tc;
2652                if (netif_attr_test_mask(j, mask, nr_ids) &&
2653                    netif_attr_test_online(j, online_mask, nr_ids)) {
2654                        /* add tx-queue to CPU/rx-queue maps */
2655                        int pos = 0;
2656
2657                        skip_tc = true;
2658
2659                        map = xmap_dereference(new_dev_maps->attr_map[tci]);
2660                        while ((pos < map->len) && (map->queues[pos] != index))
2661                                pos++;
2662
2663                        if (pos == map->len)
2664                                map->queues[map->len++] = index;
2665#ifdef CONFIG_NUMA
2666                        if (type == XPS_CPUS) {
2667                                if (numa_node_id == -2)
2668                                        numa_node_id = cpu_to_node(j);
2669                                else if (numa_node_id != cpu_to_node(j))
2670                                        numa_node_id = -1;
2671                        }
2672#endif
2673                }
2674
2675                if (copy)
2676                        xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2677                                          skip_tc);
2678        }
2679
2680        rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2681
2682        /* Cleanup old maps */
2683        if (!dev_maps)
2684                goto out_no_old_maps;
2685
2686        for (j = 0; j < dev_maps->nr_ids; j++) {
2687                for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2688                        map = xmap_dereference(dev_maps->attr_map[tci]);
2689                        if (!map)
2690                                continue;
2691
2692                        if (copy) {
2693                                new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2694                                if (map == new_map)
2695                                        continue;
2696                        }
2697
2698                        RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2699                        kfree_rcu(map, rcu);
2700                }
2701        }
2702
2703        old_dev_maps = dev_maps;
2704
2705out_no_old_maps:
2706        dev_maps = new_dev_maps;
2707        active = true;
2708
2709out_no_new_maps:
2710        if (type == XPS_CPUS)
2711                /* update Tx queue numa node */
2712                netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2713                                             (numa_node_id >= 0) ?
2714                                             numa_node_id : NUMA_NO_NODE);
2715
2716        if (!dev_maps)
2717                goto out_no_maps;
2718
2719        /* removes tx-queue from unused CPUs/rx-queues */
2720        for (j = 0; j < dev_maps->nr_ids; j++) {
2721                tci = j * dev_maps->num_tc;
2722
2723                for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2724                        if (i == tc &&
2725                            netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2726                            netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2727                                continue;
2728
2729                        active |= remove_xps_queue(dev_maps,
2730                                                   copy ? old_dev_maps : NULL,
2731                                                   tci, index);
2732                }
2733        }
2734
2735        if (old_dev_maps)
2736                kfree_rcu(old_dev_maps, rcu);
2737
2738        /* free map if not active */
2739        if (!active)
2740                reset_xps_maps(dev, dev_maps, type);
2741
2742out_no_maps:
2743        mutex_unlock(&xps_map_mutex);
2744
2745        return 0;
2746error:
2747        /* remove any maps that we added */
2748        for (j = 0; j < nr_ids; j++) {
2749                for (i = num_tc, tci = j * num_tc; i--; tci++) {
2750                        new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2751                        map = copy ?
2752                              xmap_dereference(dev_maps->attr_map[tci]) :
2753                              NULL;
2754                        if (new_map && new_map != map)
2755                                kfree(new_map);
2756                }
2757        }
2758
2759        mutex_unlock(&xps_map_mutex);
2760
2761        kfree(new_dev_maps);
2762        return -ENOMEM;
2763}
2764EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2765
2766int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2767                        u16 index)
2768{
2769        int ret;
2770
2771        cpus_read_lock();
2772        ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2773        cpus_read_unlock();
2774
2775        return ret;
2776}
2777EXPORT_SYMBOL(netif_set_xps_queue);
2778
2779#endif
2780static void netdev_unbind_all_sb_channels(struct net_device *dev)
2781{
2782        struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2783
2784        /* Unbind any subordinate channels */
2785        while (txq-- != &dev->_tx[0]) {
2786                if (txq->sb_dev)
2787                        netdev_unbind_sb_channel(dev, txq->sb_dev);
2788        }
2789}
2790
2791void netdev_reset_tc(struct net_device *dev)
2792{
2793#ifdef CONFIG_XPS
2794        netif_reset_xps_queues_gt(dev, 0);
2795#endif
2796        netdev_unbind_all_sb_channels(dev);
2797
2798        /* Reset TC configuration of device */
2799        dev->num_tc = 0;
2800        memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2801        memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2802}
2803EXPORT_SYMBOL(netdev_reset_tc);
2804
2805int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2806{
2807        if (tc >= dev->num_tc)
2808                return -EINVAL;
2809
2810#ifdef CONFIG_XPS
2811        netif_reset_xps_queues(dev, offset, count);
2812#endif
2813        dev->tc_to_txq[tc].count = count;
2814        dev->tc_to_txq[tc].offset = offset;
2815        return 0;
2816}
2817EXPORT_SYMBOL(netdev_set_tc_queue);
2818
2819int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2820{
2821        if (num_tc > TC_MAX_QUEUE)
2822                return -EINVAL;
2823
2824#ifdef CONFIG_XPS
2825        netif_reset_xps_queues_gt(dev, 0);
2826#endif
2827        netdev_unbind_all_sb_channels(dev);
2828
2829        dev->num_tc = num_tc;
2830        return 0;
2831}
2832EXPORT_SYMBOL(netdev_set_num_tc);
2833
2834void netdev_unbind_sb_channel(struct net_device *dev,
2835                              struct net_device *sb_dev)
2836{
2837        struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2838
2839#ifdef CONFIG_XPS
2840        netif_reset_xps_queues_gt(sb_dev, 0);
2841#endif
2842        memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2843        memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2844
2845        while (txq-- != &dev->_tx[0]) {
2846                if (txq->sb_dev == sb_dev)
2847                        txq->sb_dev = NULL;
2848        }
2849}
2850EXPORT_SYMBOL(netdev_unbind_sb_channel);
2851
2852int netdev_bind_sb_channel_queue(struct net_device *dev,
2853                                 struct net_device *sb_dev,
2854                                 u8 tc, u16 count, u16 offset)
2855{
2856        /* Make certain the sb_dev and dev are already configured */
2857        if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2858                return -EINVAL;
2859
2860        /* We cannot hand out queues we don't have */
2861        if ((offset + count) > dev->real_num_tx_queues)
2862                return -EINVAL;
2863
2864        /* Record the mapping */
2865        sb_dev->tc_to_txq[tc].count = count;
2866        sb_dev->tc_to_txq[tc].offset = offset;
2867
2868        /* Provide a way for Tx queue to find the tc_to_txq map or
2869         * XPS map for itself.
2870         */
2871        while (count--)
2872                netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2873
2874        return 0;
2875}
2876EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2877
2878int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2879{
2880        /* Do not use a multiqueue device to represent a subordinate channel */
2881        if (netif_is_multiqueue(dev))
2882                return -ENODEV;
2883
2884        /* We allow channels 1 - 32767 to be used for subordinate channels.
2885         * Channel 0 is meant to be "native" mode and used only to represent
2886         * the main root device. We allow writing 0 to reset the device back
2887         * to normal mode after being used as a subordinate channel.
2888         */
2889        if (channel > S16_MAX)
2890                return -EINVAL;
2891
2892        dev->num_tc = -channel;
2893
2894        return 0;
2895}
2896EXPORT_SYMBOL(netdev_set_sb_channel);
2897
2898/*
2899 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2900 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2901 */
2902int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2903{
2904        bool disabling;
2905        int rc;
2906
2907        disabling = txq < dev->real_num_tx_queues;
2908
2909        if (txq < 1 || txq > dev->num_tx_queues)
2910                return -EINVAL;
2911
2912        if (dev->reg_state == NETREG_REGISTERED ||
2913            dev->reg_state == NETREG_UNREGISTERING) {
2914                ASSERT_RTNL();
2915
2916                rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2917                                                  txq);
2918                if (rc)
2919                        return rc;
2920
2921                if (dev->num_tc)
2922                        netif_setup_tc(dev, txq);
2923
2924                dev->real_num_tx_queues = txq;
2925
2926                if (disabling) {
2927                        synchronize_net();
2928                        qdisc_reset_all_tx_gt(dev, txq);
2929#ifdef CONFIG_XPS
2930                        netif_reset_xps_queues_gt(dev, txq);
2931#endif
2932                }
2933        } else {
2934                dev->real_num_tx_queues = txq;
2935        }
2936
2937        return 0;
2938}
2939EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2940
2941#ifdef CONFIG_SYSFS
2942/**
2943 *      netif_set_real_num_rx_queues - set actual number of RX queues used
2944 *      @dev: Network device
2945 *      @rxq: Actual number of RX queues
2946 *
2947 *      This must be called either with the rtnl_lock held or before
2948 *      registration of the net device.  Returns 0 on success, or a
2949 *      negative error code.  If called before registration, it always
2950 *      succeeds.
2951 */
2952int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2953{
2954        int rc;
2955
2956        if (rxq < 1 || rxq > dev->num_rx_queues)
2957                return -EINVAL;
2958
2959        if (dev->reg_state == NETREG_REGISTERED) {
2960                ASSERT_RTNL();
2961
2962                rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2963                                                  rxq);
2964                if (rc)
2965                        return rc;
2966        }
2967
2968        dev->real_num_rx_queues = rxq;
2969        return 0;
2970}
2971EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2972#endif
2973
2974/**
2975 *      netif_set_real_num_queues - set actual number of RX and TX queues used
2976 *      @dev: Network device
2977 *      @txq: Actual number of TX queues
2978 *      @rxq: Actual number of RX queues
2979 *
2980 *      Set the real number of both TX and RX queues.
2981 *      Does nothing if the number of queues is already correct.
2982 */
2983int netif_set_real_num_queues(struct net_device *dev,
2984                              unsigned int txq, unsigned int rxq)
2985{
2986        unsigned int old_rxq = dev->real_num_rx_queues;
2987        int err;
2988
2989        if (txq < 1 || txq > dev->num_tx_queues ||
2990            rxq < 1 || rxq > dev->num_rx_queues)
2991                return -EINVAL;
2992
2993        /* Start from increases, so the error path only does decreases -
2994         * decreases can't fail.
2995         */
2996        if (rxq > dev->real_num_rx_queues) {
2997                err = netif_set_real_num_rx_queues(dev, rxq);
2998                if (err)
2999                        return err;
3000        }
3001        if (txq > dev->real_num_tx_queues) {
3002                err = netif_set_real_num_tx_queues(dev, txq);
3003                if (err)
3004                        goto undo_rx;
3005        }
3006        if (rxq < dev->real_num_rx_queues)
3007                WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3008        if (txq < dev->real_num_tx_queues)
3009                WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3010
3011        return 0;
3012undo_rx:
3013        WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3014        return err;
3015}
3016EXPORT_SYMBOL(netif_set_real_num_queues);
3017
3018/**
3019 * netif_get_num_default_rss_queues - default number of RSS queues
3020 *
3021 * This routine should set an upper limit on the number of RSS queues
3022 * used by default by multiqueue devices.
3023 */
3024int netif_get_num_default_rss_queues(void)
3025{
3026        return is_kdump_kernel() ?
3027                1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
3028}
3029EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3030
3031static void __netif_reschedule(struct Qdisc *q)
3032{
3033        struct softnet_data *sd;
3034        unsigned long flags;
3035
3036        local_irq_save(flags);
3037        sd = this_cpu_ptr(&softnet_data);
3038        q->next_sched = NULL;
3039        *sd->output_queue_tailp = q;
3040        sd->output_queue_tailp = &q->next_sched;
3041        raise_softirq_irqoff(NET_TX_SOFTIRQ);
3042        local_irq_restore(flags);
3043}
3044
3045void __netif_schedule(struct Qdisc *q)
3046{
3047        if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3048                __netif_reschedule(q);
3049}
3050EXPORT_SYMBOL(__netif_schedule);
3051
3052struct dev_kfree_skb_cb {
3053        enum skb_free_reason reason;
3054};
3055
3056static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3057{
3058        return (struct dev_kfree_skb_cb *)skb->cb;
3059}
3060
3061void netif_schedule_queue(struct netdev_queue *txq)
3062{
3063        rcu_read_lock();
3064        if (!netif_xmit_stopped(txq)) {
3065                struct Qdisc *q = rcu_dereference(txq->qdisc);
3066
3067                __netif_schedule(q);
3068        }
3069        rcu_read_unlock();
3070}
3071EXPORT_SYMBOL(netif_schedule_queue);
3072
3073void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3074{
3075        if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3076                struct Qdisc *q;
3077
3078                rcu_read_lock();
3079                q = rcu_dereference(dev_queue->qdisc);
3080                __netif_schedule(q);
3081                rcu_read_unlock();
3082        }
3083}
3084EXPORT_SYMBOL(netif_tx_wake_queue);
3085
3086void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3087{
3088        unsigned long flags;
3089
3090        if (unlikely(!skb))
3091                return;
3092
3093        if (likely(refcount_read(&skb->users) == 1)) {
3094                smp_rmb();
3095                refcount_set(&skb->users, 0);
3096        } else if (likely(!refcount_dec_and_test(&skb->users))) {
3097                return;
3098        }
3099        get_kfree_skb_cb(skb)->reason = reason;
3100        local_irq_save(flags);
3101        skb->next = __this_cpu_read(softnet_data.completion_queue);
3102        __this_cpu_write(softnet_data.completion_queue, skb);
3103        raise_softirq_irqoff(NET_TX_SOFTIRQ);
3104        local_irq_restore(flags);
3105}
3106EXPORT_SYMBOL(__dev_kfree_skb_irq);
3107
3108void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
3109{
3110        if (in_hardirq() || irqs_disabled())
3111                __dev_kfree_skb_irq(skb, reason);
3112        else
3113                dev_kfree_skb(skb);
3114}
3115EXPORT_SYMBOL(__dev_kfree_skb_any);
3116
3117
3118/**
3119 * netif_device_detach - mark device as removed
3120 * @dev: network device
3121 *
3122 * Mark device as removed from system and therefore no longer available.
3123 */
3124void netif_device_detach(struct net_device *dev)
3125{
3126        if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3127            netif_running(dev)) {
3128                netif_tx_stop_all_queues(dev);
3129        }
3130}
3131EXPORT_SYMBOL(netif_device_detach);
3132
3133/**
3134 * netif_device_attach - mark device as attached
3135 * @dev: network device
3136 *
3137 * Mark device as attached from system and restart if needed.
3138 */
3139void netif_device_attach(struct net_device *dev)
3140{
3141        if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3142            netif_running(dev)) {
3143                netif_tx_wake_all_queues(dev);
3144                __netdev_watchdog_up(dev);
3145        }
3146}
3147EXPORT_SYMBOL(netif_device_attach);
3148
3149/*
3150 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3151 * to be used as a distribution range.
3152 */
3153static u16 skb_tx_hash(const struct net_device *dev,
3154                       const struct net_device *sb_dev,
3155                       struct sk_buff *skb)
3156{
3157        u32 hash;
3158        u16 qoffset = 0;
3159        u16 qcount = dev->real_num_tx_queues;
3160
3161        if (dev->num_tc) {
3162                u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3163
3164                qoffset = sb_dev->tc_to_txq[tc].offset;
3165                qcount = sb_dev->tc_to_txq[tc].count;
3166                if (unlikely(!qcount)) {
3167                        net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3168                                             sb_dev->name, qoffset, tc);
3169                        qoffset = 0;
3170                        qcount = dev->real_num_tx_queues;
3171                }
3172        }
3173
3174        if (skb_rx_queue_recorded(skb)) {
3175                hash = skb_get_rx_queue(skb);
3176                if (hash >= qoffset)
3177                        hash -= qoffset;
3178                while (unlikely(hash >= qcount))
3179                        hash -= qcount;
3180                return hash + qoffset;
3181        }
3182
3183        return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3184}
3185
3186static void skb_warn_bad_offload(const struct sk_buff *skb)
3187{
3188        static const netdev_features_t null_features;
3189        struct net_device *dev = skb->dev;
3190        const char *name = "";
3191
3192        if (!net_ratelimit())
3193                return;
3194
3195        if (dev) {
3196                if (dev->dev.parent)
3197                        name = dev_driver_string(dev->dev.parent);
3198                else
3199                        name = netdev_name(dev);
3200        }
3201        skb_dump(KERN_WARNING, skb, false);
3202        WARN(1, "%s: caps=(%pNF, %pNF)\n",
3203             name, dev ? &dev->features : &null_features,
3204             skb->sk ? &skb->sk->sk_route_caps : &null_features);
3205}
3206
3207/*
3208 * Invalidate hardware checksum when packet is to be mangled, and
3209 * complete checksum manually on outgoing path.
3210 */
3211int skb_checksum_help(struct sk_buff *skb)
3212{
3213        __wsum csum;
3214        int ret = 0, offset;
3215
3216        if (skb->ip_summed == CHECKSUM_COMPLETE)
3217                goto out_set_summed;
3218
3219        if (unlikely(skb_is_gso(skb))) {
3220                skb_warn_bad_offload(skb);
3221                return -EINVAL;
3222        }
3223
3224        /* Before computing a checksum, we should make sure no frag could
3225         * be modified by an external entity : checksum could be wrong.
3226         */
3227        if (skb_has_shared_frag(skb)) {
3228                ret = __skb_linearize(skb);
3229                if (ret)
3230                        goto out;
3231        }
3232
3233        offset = skb_checksum_start_offset(skb);
3234        BUG_ON(offset >= skb_headlen(skb));
3235        csum = skb_checksum(skb, offset, skb->len - offset, 0);
3236
3237        offset += skb->csum_offset;
3238        BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
3239
3240        ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3241        if (ret)
3242                goto out;
3243
3244        *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3245out_set_summed:
3246        skb->ip_summed = CHECKSUM_NONE;
3247out:
3248        return ret;
3249}
3250EXPORT_SYMBOL(skb_checksum_help);
3251
3252int skb_crc32c_csum_help(struct sk_buff *skb)
3253{
3254        __le32 crc32c_csum;
3255        int ret = 0, offset, start;
3256
3257        if (skb->ip_summed != CHECKSUM_PARTIAL)
3258                goto out;
3259
3260        if (unlikely(skb_is_gso(skb)))
3261                goto out;
3262
3263        /* Before computing a checksum, we should make sure no frag could
3264         * be modified by an external entity : checksum could be wrong.
3265         */
3266        if (unlikely(skb_has_shared_frag(skb))) {
3267                ret = __skb_linearize(skb);
3268                if (ret)
3269                        goto out;
3270        }
3271        start = skb_checksum_start_offset(skb);
3272        offset = start + offsetof(struct sctphdr, checksum);
3273        if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3274                ret = -EINVAL;
3275                goto out;
3276        }
3277
3278        ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3279        if (ret)
3280                goto out;
3281
3282        crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3283                                                  skb->len - start, ~(__u32)0,
3284                                                  crc32c_csum_stub));
3285        *(__le32 *)(skb->data + offset) = crc32c_csum;
3286        skb->ip_summed = CHECKSUM_NONE;
3287        skb->csum_not_inet = 0;
3288out:
3289        return ret;
3290}
3291
3292__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3293{
3294        __be16 type = skb->protocol;
3295
3296        /* Tunnel gso handlers can set protocol to ethernet. */
3297        if (type == htons(ETH_P_TEB)) {
3298                struct ethhdr *eth;
3299
3300                if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3301                        return 0;
3302
3303                eth = (struct ethhdr *)skb->data;
3304                type = eth->h_proto;
3305        }
3306
3307        return __vlan_get_protocol(skb, type, depth);
3308}
3309
3310/**
3311 *      skb_mac_gso_segment - mac layer segmentation handler.
3312 *      @skb: buffer to segment
3313 *      @features: features for the output path (see dev->features)
3314 */
3315struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3316                                    netdev_features_t features)
3317{
3318        struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3319        struct packet_offload *ptype;
3320        int vlan_depth = skb->mac_len;
3321        __be16 type = skb_network_protocol(skb, &vlan_depth);
3322
3323        if (unlikely(!type))
3324                return ERR_PTR(-EINVAL);
3325
3326        __skb_pull(skb, vlan_depth);
3327
3328        rcu_read_lock();
3329        list_for_each_entry_rcu(ptype, &offload_base, list) {
3330                if (ptype->type == type && ptype->callbacks.gso_segment) {
3331                        segs = ptype->callbacks.gso_segment(skb, features);
3332                        break;
3333                }
3334        }
3335        rcu_read_unlock();
3336
3337        __skb_push(skb, skb->data - skb_mac_header(skb));
3338
3339        return segs;
3340}
3341EXPORT_SYMBOL(skb_mac_gso_segment);
3342
3343
3344/* openvswitch calls this on rx path, so we need a different check.
3345 */
3346static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3347{
3348        if (tx_path)
3349                return skb->ip_summed != CHECKSUM_PARTIAL &&
3350                       skb->ip_summed != CHECKSUM_UNNECESSARY;
3351
3352        return skb->ip_summed == CHECKSUM_NONE;
3353}
3354
3355/**
3356 *      __skb_gso_segment - Perform segmentation on skb.
3357 *      @skb: buffer to segment
3358 *      @features: features for the output path (see dev->features)
3359 *      @tx_path: whether it is called in TX path
3360 *
3361 *      This function segments the given skb and returns a list of segments.
3362 *
3363 *      It may return NULL if the skb requires no segmentation.  This is
3364 *      only possible when GSO is used for verifying header integrity.
3365 *
3366 *      Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3367 */
3368struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3369                                  netdev_features_t features, bool tx_path)
3370{
3371        struct sk_buff *segs;
3372
3373        if (unlikely(skb_needs_check(skb, tx_path))) {
3374                int err;
3375
3376                /* We're going to init ->check field in TCP or UDP header */
3377                err = skb_cow_head(skb, 0);
3378                if (err < 0)
3379                        return ERR_PTR(err);
3380        }
3381
3382        /* Only report GSO partial support if it will enable us to
3383         * support segmentation on this frame without needing additional
3384         * work.
3385         */
3386        if (features & NETIF_F_GSO_PARTIAL) {
3387                netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3388                struct net_device *dev = skb->dev;
3389
3390                partial_features |= dev->features & dev->gso_partial_features;
3391                if (!skb_gso_ok(skb, features | partial_features))
3392                        features &= ~NETIF_F_GSO_PARTIAL;
3393        }
3394
3395        BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3396                     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3397
3398        SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3399        SKB_GSO_CB(skb)->encap_level = 0;
3400
3401        skb_reset_mac_header(skb);
3402        skb_reset_mac_len(skb);
3403
3404        segs = skb_mac_gso_segment(skb, features);
3405
3406        if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3407                skb_warn_bad_offload(skb);
3408
3409        return segs;
3410}
3411EXPORT_SYMBOL(__skb_gso_segment);
3412
3413/* Take action when hardware reception checksum errors are detected. */
3414#ifdef CONFIG_BUG
3415static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3416{
3417        pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3418        skb_dump(KERN_ERR, skb, true);
3419        dump_stack();
3420}
3421
3422void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3423{
3424        DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3425}
3426EXPORT_SYMBOL(netdev_rx_csum_fault);
3427#endif
3428
3429/* XXX: check that highmem exists at all on the given machine. */
3430static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3431{
3432#ifdef CONFIG_HIGHMEM
3433        int i;
3434
3435        if (!(dev->features & NETIF_F_HIGHDMA)) {
3436                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3437                        skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3438
3439                        if (PageHighMem(skb_frag_page(frag)))
3440                                return 1;
3441                }
3442        }
3443#endif
3444        return 0;
3445}
3446
3447/* If MPLS offload request, verify we are testing hardware MPLS features
3448 * instead of standard features for the netdev.
3449 */
3450#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3451static netdev_features_t net_mpls_features(struct sk_buff *skb,
3452                                           netdev_features_t features,
3453                                           __be16 type)
3454{
3455        if (eth_p_mpls(type))
3456                features &= skb->dev->mpls_features;
3457
3458        return features;
3459}
3460#else
3461static netdev_features_t net_mpls_features(struct sk_buff *skb,
3462                                           netdev_features_t features,
3463                                           __be16 type)
3464{
3465        return features;
3466}
3467#endif
3468
3469static netdev_features_t harmonize_features(struct sk_buff *skb,
3470        netdev_features_t features)
3471{
3472        __be16 type;
3473
3474        type = skb_network_protocol(skb, NULL);
3475        features = net_mpls_features(skb, features, type);
3476
3477        if (skb->ip_summed != CHECKSUM_NONE &&
3478            !can_checksum_protocol(features, type)) {
3479                features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3480        }
3481        if (illegal_highdma(skb->dev, skb))
3482                features &= ~NETIF_F_SG;
3483
3484        return features;
3485}
3486
3487netdev_features_t passthru_features_check(struct sk_buff *skb,
3488                                          struct net_device *dev,
3489                                          netdev_features_t features)
3490{
3491        return features;
3492}
3493EXPORT_SYMBOL(passthru_features_check);
3494
3495static netdev_features_t dflt_features_check(struct sk_buff *skb,
3496                                             struct net_device *dev,
3497                                             netdev_features_t features)
3498{
3499        return vlan_features_check(skb, features);
3500}
3501
3502static netdev_features_t gso_features_check(const struct sk_buff *skb,
3503                                            struct net_device *dev,
3504                                            netdev_features_t features)
3505{
3506        u16 gso_segs = skb_shinfo(skb)->gso_segs;
3507
3508        if (gso_segs > dev->gso_max_segs)
3509                return features & ~NETIF_F_GSO_MASK;
3510
3511        if (!skb_shinfo(skb)->gso_type) {
3512                skb_warn_bad_offload(skb);
3513                return features & ~NETIF_F_GSO_MASK;
3514        }
3515
3516        /* Support for GSO partial features requires software
3517         * intervention before we can actually process the packets
3518         * so we need to strip support for any partial features now
3519         * and we can pull them back in after we have partially
3520         * segmented the frame.
3521         */
3522        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3523                features &= ~dev->gso_partial_features;
3524
3525        /* Make sure to clear the IPv4 ID mangling feature if the
3526         * IPv4 header has the potential to be fragmented.
3527         */
3528        if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3529                struct iphdr *iph = skb->encapsulation ?
3530                                    inner_ip_hdr(skb) : ip_hdr(skb);
3531
3532                if (!(iph->frag_off & htons(IP_DF)))
3533                        features &= ~NETIF_F_TSO_MANGLEID;
3534        }
3535
3536        return features;
3537}
3538
3539netdev_features_t netif_skb_features(struct sk_buff *skb)
3540{
3541        struct net_device *dev = skb->dev;
3542        netdev_features_t features = dev->features;
3543
3544        if (skb_is_gso(skb))
3545                features = gso_features_check(skb, dev, features);
3546
3547        /* If encapsulation offload request, verify we are testing
3548         * hardware encapsulation features instead of standard
3549         * features for the netdev
3550         */
3551        if (skb->encapsulation)
3552                features &= dev->hw_enc_features;
3553
3554        if (skb_vlan_tagged(skb))
3555                features = netdev_intersect_features(features,
3556                                                     dev->vlan_features |
3557                                                     NETIF_F_HW_VLAN_CTAG_TX |
3558                                                     NETIF_F_HW_VLAN_STAG_TX);
3559
3560        if (dev->netdev_ops->ndo_features_check)
3561                features &= dev->netdev_ops->ndo_features_check(skb, dev,
3562                                                                features);
3563        else
3564                features &= dflt_features_check(skb, dev, features);
3565
3566        return harmonize_features(skb, features);
3567}
3568EXPORT_SYMBOL(netif_skb_features);
3569
3570static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3571                    struct netdev_queue *txq, bool more)
3572{
3573        unsigned int len;
3574        int rc;
3575
3576        if (dev_nit_active(dev))
3577                dev_queue_xmit_nit(skb, dev);
3578
3579        len = skb->len;
3580        PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
3581        trace_net_dev_start_xmit(skb, dev);
3582        rc = netdev_start_xmit(skb, dev, txq, more);
3583        trace_net_dev_xmit(skb, rc, dev, len);
3584
3585        return rc;
3586}
3587
3588struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3589                                    struct netdev_queue *txq, int *ret)
3590{
3591        struct sk_buff *skb = first;
3592        int rc = NETDEV_TX_OK;
3593
3594        while (skb) {
3595                struct sk_buff *next = skb->next;
3596
3597                skb_mark_not_on_list(skb);
3598                rc = xmit_one(skb, dev, txq, next != NULL);
3599                if (unlikely(!dev_xmit_complete(rc))) {
3600                        skb->next = next;
3601                        goto out;
3602                }
3603
3604                skb = next;
3605                if (netif_tx_queue_stopped(txq) && skb) {
3606                        rc = NETDEV_TX_BUSY;
3607                        break;
3608                }
3609        }
3610
3611out:
3612        *ret = rc;
3613        return skb;
3614}
3615
3616static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3617                                          netdev_features_t features)
3618{
3619        if (skb_vlan_tag_present(skb) &&
3620            !vlan_hw_offload_capable(features, skb->vlan_proto))
3621                skb = __vlan_hwaccel_push_inside(skb);
3622        return skb;
3623}
3624
3625int skb_csum_hwoffload_help(struct sk_buff *skb,
3626                            const netdev_features_t features)
3627{
3628        if (unlikely(skb_csum_is_sctp(skb)))
3629                return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3630                        skb_crc32c_csum_help(skb);
3631
3632        if (features & NETIF_F_HW_CSUM)
3633                return 0;
3634
3635        if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3636                switch (skb->csum_offset) {
3637                case offsetof(struct tcphdr, check):
3638                case offsetof(struct udphdr, check):
3639                        return 0;
3640                }
3641        }
3642
3643        return skb_checksum_help(skb);
3644}
3645EXPORT_SYMBOL(skb_csum_hwoffload_help);
3646
3647static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3648{
3649        netdev_features_t features;
3650
3651        features = netif_skb_features(skb);
3652        skb = validate_xmit_vlan(skb, features);
3653        if (unlikely(!skb))
3654                goto out_null;
3655
3656        skb = sk_validate_xmit_skb(skb, dev);
3657        if (unlikely(!skb))
3658                goto out_null;
3659
3660        if (netif_needs_gso(skb, features)) {
3661                struct sk_buff *segs;
3662
3663                segs = skb_gso_segment(skb, features);
3664                if (IS_ERR(segs)) {
3665                        goto out_kfree_skb;
3666                } else if (segs) {
3667                        consume_skb(skb);
3668                        skb = segs;
3669                }
3670        } else {
3671                if (skb_needs_linearize(skb, features) &&
3672                    __skb_linearize(skb))
3673                        goto out_kfree_skb;
3674
3675                /* If packet is not checksummed and device does not
3676                 * support checksumming for this protocol, complete
3677                 * checksumming here.
3678                 */
3679                if (skb->ip_summed == CHECKSUM_PARTIAL) {
3680                        if (skb->encapsulation)
3681                                skb_set_inner_transport_header(skb,
3682                                                               skb_checksum_start_offset(skb));
3683                        else
3684                                skb_set_transport_header(skb,
3685                                                         skb_checksum_start_offset(skb));
3686                        if (skb_csum_hwoffload_help(skb, features))
3687                                goto out_kfree_skb;
3688                }
3689        }
3690
3691        skb = validate_xmit_xfrm(skb, features, again);
3692
3693        return skb;
3694
3695out_kfree_skb:
3696        kfree_skb(skb);
3697out_null:
3698        atomic_long_inc(&dev->tx_dropped);
3699        return NULL;
3700}
3701
3702struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3703{
3704        struct sk_buff *next, *head = NULL, *tail;
3705
3706        for (; skb != NULL; skb = next) {
3707                next = skb->next;
3708                skb_mark_not_on_list(skb);
3709
3710                /* in case skb wont be segmented, point to itself */
3711                skb->prev = skb;
3712
3713                skb = validate_xmit_skb(skb, dev, again);
3714                if (!skb)
3715                        continue;
3716
3717                if (!head)
3718                        head = skb;
3719                else
3720                        tail->next = skb;
3721                /* If skb was segmented, skb->prev points to
3722                 * the last segment. If not, it still contains skb.
3723                 */
3724                tail = skb->prev;
3725        }
3726        return head;
3727}
3728EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3729
3730static void qdisc_pkt_len_init(struct sk_buff *skb)
3731{
3732        const struct skb_shared_info *shinfo = skb_shinfo(skb);
3733
3734        qdisc_skb_cb(skb)->pkt_len = skb->len;
3735
3736        /* To get more precise estimation of bytes sent on wire,
3737         * we add to pkt_len the headers size of all segments
3738         */
3739        if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3740                unsigned int hdr_len;
3741                u16 gso_segs = shinfo->gso_segs;
3742
3743                /* mac layer + network layer */
3744                hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3745
3746                /* + transport layer */
3747                if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3748                        const struct tcphdr *th;
3749                        struct tcphdr _tcphdr;
3750
3751                        th = skb_header_pointer(skb, skb_transport_offset(skb),
3752                                                sizeof(_tcphdr), &_tcphdr);
3753                        if (likely(th))
3754                                hdr_len += __tcp_hdrlen(th);
3755                } else {
3756                        struct udphdr _udphdr;
3757
3758                        if (skb_header_pointer(skb, skb_transport_offset(skb),
3759                                               sizeof(_udphdr), &_udphdr))
3760                                hdr_len += sizeof(struct udphdr);
3761                }
3762
3763                if (shinfo->gso_type & SKB_GSO_DODGY)
3764                        gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3765                                                shinfo->gso_size);
3766
3767                qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3768        }
3769}
3770
3771static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3772                             struct sk_buff **to_free,
3773                             struct netdev_queue *txq)
3774{
3775        int rc;
3776
3777        rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3778        if (rc == NET_XMIT_SUCCESS)
3779                trace_qdisc_enqueue(q, txq, skb);
3780        return rc;
3781}
3782
3783static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3784                                 struct net_device *dev,
3785                                 struct netdev_queue *txq)
3786{
3787        spinlock_t *root_lock = qdisc_lock(q);
3788        struct sk_buff *to_free = NULL;
3789        bool contended;
3790        int rc;
3791
3792        qdisc_calculate_pkt_len(skb, q);
3793
3794        if (q->flags & TCQ_F_NOLOCK) {
3795                if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3796                    qdisc_run_begin(q)) {
3797                        /* Retest nolock_qdisc_is_empty() within the protection
3798                         * of q->seqlock to protect from racing with requeuing.
3799                         */
3800                        if (unlikely(!nolock_qdisc_is_empty(q))) {
3801                                rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3802                                __qdisc_run(q);
3803                                qdisc_run_end(q);
3804
3805                                goto no_lock_out;
3806                        }
3807
3808                        qdisc_bstats_cpu_update(q, skb);
3809                        if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3810                            !nolock_qdisc_is_empty(q))
3811                                __qdisc_run(q);
3812
3813                        qdisc_run_end(q);
3814                        return NET_XMIT_SUCCESS;
3815                }
3816
3817                rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3818                qdisc_run(q);
3819
3820no_lock_out:
3821                if (unlikely(to_free))
3822                        kfree_skb_list(to_free);
3823                return rc;
3824        }
3825
3826        /*
3827         * Heuristic to force contended enqueues to serialize on a
3828         * separate lock before trying to get qdisc main lock.
3829         * This permits qdisc->running owner to get the lock more
3830         * often and dequeue packets faster.
3831         */
3832        contended = qdisc_is_running(q);
3833        if (unlikely(contended))
3834                spin_lock(&q->busylock);
3835
3836        spin_lock(root_lock);
3837        if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3838                __qdisc_drop(skb, &to_free);
3839                rc = NET_XMIT_DROP;
3840        } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3841                   qdisc_run_begin(q)) {
3842                /*
3843                 * This is a work-conserving queue; there are no old skbs
3844                 * waiting to be sent out; and the qdisc is not running -
3845                 * xmit the skb directly.
3846                 */
3847
3848                qdisc_bstats_update(q, skb);
3849
3850                if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3851                        if (unlikely(contended)) {
3852                                spin_unlock(&q->busylock);
3853                                contended = false;
3854                        }
3855                        __qdisc_run(q);
3856                }
3857
3858                qdisc_run_end(q);
3859                rc = NET_XMIT_SUCCESS;
3860        } else {
3861                rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3862                if (qdisc_run_begin(q)) {
3863                        if (unlikely(contended)) {
3864                                spin_unlock(&q->busylock);
3865                                contended = false;
3866                        }
3867                        __qdisc_run(q);
3868                        qdisc_run_end(q);
3869                }
3870        }
3871        spin_unlock(root_lock);
3872        if (unlikely(to_free))
3873                kfree_skb_list(to_free);
3874        if (unlikely(contended))
3875                spin_unlock(&q->busylock);
3876        return rc;
3877}
3878
3879#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3880static void skb_update_prio(struct sk_buff *skb)
3881{
3882        const struct netprio_map *map;
3883        const struct sock *sk;
3884        unsigned int prioidx;
3885
3886        if (skb->priority)
3887                return;
3888        map = rcu_dereference_bh(skb->dev->priomap);
3889        if (!map)
3890                return;
3891        sk = skb_to_full_sk(skb);
3892        if (!sk)
3893                return;
3894
3895        prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3896
3897        if (prioidx < map->priomap_len)
3898                skb->priority = map->priomap[prioidx];
3899}
3900#else
3901#define skb_update_prio(skb)
3902#endif
3903
3904/**
3905 *      dev_loopback_xmit - loop back @skb
3906 *      @net: network namespace this loopback is happening in
3907 *      @sk:  sk needed to be a netfilter okfn
3908 *      @skb: buffer to transmit
3909 */
3910int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3911{
3912        skb_reset_mac_header(skb);
3913        __skb_pull(skb, skb_network_offset(skb));
3914        skb->pkt_type = PACKET_LOOPBACK;
3915        if (skb->ip_summed == CHECKSUM_NONE)
3916                skb->ip_summed = CHECKSUM_UNNECESSARY;
3917        WARN_ON(!skb_dst(skb));
3918        skb_dst_force(skb);
3919        netif_rx_ni(skb);
3920        return 0;
3921}
3922EXPORT_SYMBOL(dev_loopback_xmit);
3923
3924#ifdef CONFIG_NET_EGRESS
3925static struct sk_buff *
3926sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3927{
3928        struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3929        struct tcf_result cl_res;
3930
3931        if (!miniq)
3932                return skb;
3933
3934        /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3935        qdisc_skb_cb(skb)->mru = 0;
3936        qdisc_skb_cb(skb)->post_ct = false;
3937        mini_qdisc_bstats_cpu_update(miniq, skb);
3938
3939        switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3940        case TC_ACT_OK:
3941        case TC_ACT_RECLASSIFY:
3942                skb->tc_index = TC_H_MIN(cl_res.classid);
3943                break;
3944        case TC_ACT_SHOT:
3945                mini_qdisc_qstats_cpu_drop(miniq);
3946                *ret = NET_XMIT_DROP;
3947                kfree_skb(skb);
3948                return NULL;
3949        case TC_ACT_STOLEN:
3950        case TC_ACT_QUEUED:
3951        case TC_ACT_TRAP:
3952                *ret = NET_XMIT_SUCCESS;
3953                consume_skb(skb);
3954                return NULL;
3955        case TC_ACT_REDIRECT:
3956                /* No need to push/pop skb's mac_header here on egress! */
3957                skb_do_redirect(skb);
3958                *ret = NET_XMIT_SUCCESS;
3959                return NULL;
3960        default:
3961                break;
3962        }
3963
3964        return skb;
3965}
3966#endif /* CONFIG_NET_EGRESS */
3967
3968#ifdef CONFIG_XPS
3969static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3970                               struct xps_dev_maps *dev_maps, unsigned int tci)
3971{
3972        int tc = netdev_get_prio_tc_map(dev, skb->priority);
3973        struct xps_map *map;
3974        int queue_index = -1;
3975
3976        if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
3977                return queue_index;
3978
3979        tci *= dev_maps->num_tc;
3980        tci += tc;
3981
3982        map = rcu_dereference(dev_maps->attr_map[tci]);
3983        if (map) {
3984                if (map->len == 1)
3985                        queue_index = map->queues[0];
3986                else
3987                        queue_index = map->queues[reciprocal_scale(
3988                                                skb_get_hash(skb), map->len)];
3989                if (unlikely(queue_index >= dev->real_num_tx_queues))
3990                        queue_index = -1;
3991        }
3992        return queue_index;
3993}
3994#endif
3995
3996static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
3997                         struct sk_buff *skb)
3998{
3999#ifdef CONFIG_XPS
4000        struct xps_dev_maps *dev_maps;
4001        struct sock *sk = skb->sk;
4002        int queue_index = -1;
4003
4004        if (!static_key_false(&xps_needed))
4005                return -1;
4006
4007        rcu_read_lock();
4008        if (!static_key_false(&xps_rxqs_needed))
4009                goto get_cpus_map;
4010
4011        dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4012        if (dev_maps) {
4013                int tci = sk_rx_queue_get(sk);
4014
4015                if (tci >= 0)
4016                        queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4017                                                          tci);
4018        }
4019
4020get_cpus_map:
4021        if (queue_index < 0) {
4022                dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4023                if (dev_maps) {
4024                        unsigned int tci = skb->sender_cpu - 1;
4025
4026                        queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4027                                                          tci);
4028                }
4029        }
4030        rcu_read_unlock();
4031
4032        return queue_index;
4033#else
4034        return -1;
4035#endif
4036}
4037
4038u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4039                     struct net_device *sb_dev)
4040{
4041        return 0;
4042}
4043EXPORT_SYMBOL(dev_pick_tx_zero);
4044
4045u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4046                       struct net_device *sb_dev)
4047{
4048        return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4049}
4050EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4051
4052u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4053                     struct net_device *sb_dev)
4054{
4055        struct sock *sk = skb->sk;
4056        int queue_index = sk_tx_queue_get(sk);
4057
4058        sb_dev = sb_dev ? : dev;
4059
4060        if (queue_index < 0 || skb->ooo_okay ||
4061            queue_index >= dev->real_num_tx_queues) {
4062                int new_index = get_xps_queue(dev, sb_dev, skb);
4063
4064                if (new_index < 0)
4065                        new_index = skb_tx_hash(dev, sb_dev, skb);
4066
4067                if (queue_index != new_index && sk &&
4068                    sk_fullsock(sk) &&
4069                    rcu_access_pointer(sk->sk_dst_cache))
4070                        sk_tx_queue_set(sk, new_index);
4071
4072                queue_index = new_index;
4073        }
4074
4075        return queue_index;
4076}
4077EXPORT_SYMBOL(netdev_pick_tx);
4078
4079struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4080                                         struct sk_buff *skb,
4081                                         struct net_device *sb_dev)
4082{
4083        int queue_index = 0;
4084
4085#ifdef CONFIG_XPS
4086        u32 sender_cpu = skb->sender_cpu - 1;
4087
4088        if (sender_cpu >= (u32)NR_CPUS)
4089                skb->sender_cpu = raw_smp_processor_id() + 1;
4090#endif
4091
4092        if (dev->real_num_tx_queues != 1) {
4093                const struct net_device_ops *ops = dev->netdev_ops;
4094
4095                if (ops->ndo_select_queue)
4096                        queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4097                else
4098                        queue_index = netdev_pick_tx(dev, skb, sb_dev);
4099
4100                queue_index = netdev_cap_txqueue(dev, queue_index);
4101        }
4102
4103        skb_set_queue_mapping(skb, queue_index);
4104        return netdev_get_tx_queue(dev, queue_index);
4105}
4106
4107/**
4108 *      __dev_queue_xmit - transmit a buffer
4109 *      @skb: buffer to transmit
4110 *      @sb_dev: suboordinate device used for L2 forwarding offload
4111 *
4112 *      Queue a buffer for transmission to a network device. The caller must
4113 *      have set the device and priority and built the buffer before calling
4114 *      this function. The function can be called from an interrupt.
4115 *
4116 *      A negative errno code is returned on a failure. A success does not
4117 *      guarantee the frame will be transmitted as it may be dropped due
4118 *      to congestion or traffic shaping.
4119 *
4120 * -----------------------------------------------------------------------------------
4121 *      I notice this method can also return errors from the queue disciplines,
4122 *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
4123 *      be positive.
4124 *
4125 *      Regardless of the return value, the skb is consumed, so it is currently
4126 *      difficult to retry a send to this method.  (You can bump the ref count
4127 *      before sending to hold a reference for retry if you are careful.)
4128 *
4129 *      When calling this method, interrupts MUST be enabled.  This is because
4130 *      the BH enable code must have IRQs enabled so that it will not deadlock.
4131 *          --BLG
4132 */
4133static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4134{
4135        struct net_device *dev = skb->dev;
4136        struct netdev_queue *txq;
4137        struct Qdisc *q;
4138        int rc = -ENOMEM;
4139        bool again = false;
4140
4141        skb_reset_mac_header(skb);
4142
4143        if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4144                __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4145
4146        /* Disable soft irqs for various locks below. Also
4147         * stops preemption for RCU.
4148         */
4149        rcu_read_lock_bh();
4150
4151        skb_update_prio(skb);
4152
4153        qdisc_pkt_len_init(skb);
4154#ifdef CONFIG_NET_CLS_ACT
4155        skb->tc_at_ingress = 0;
4156# ifdef CONFIG_NET_EGRESS
4157        if (static_branch_unlikely(&egress_needed_key)) {
4158                skb = sch_handle_egress(skb, &rc, dev);
4159                if (!skb)
4160                        goto out;
4161        }
4162# endif
4163#endif
4164        /* If device/qdisc don't need skb->dst, release it right now while
4165         * its hot in this cpu cache.
4166         */
4167        if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4168                skb_dst_drop(skb);
4169        else
4170                skb_dst_force(skb);
4171
4172        txq = netdev_core_pick_tx(dev, skb, sb_dev);
4173        q = rcu_dereference_bh(txq->qdisc);
4174
4175        trace_net_dev_queue(skb);
4176        if (q->enqueue) {
4177                rc = __dev_xmit_skb(skb, q, dev, txq);
4178                goto out;
4179        }
4180
4181        /* The device has no queue. Common case for software devices:
4182         * loopback, all the sorts of tunnels...
4183
4184         * Really, it is unlikely that netif_tx_lock protection is necessary
4185         * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
4186         * counters.)
4187         * However, it is possible, that they rely on protection
4188         * made by us here.
4189
4190         * Check this and shot the lock. It is not prone from deadlocks.
4191         *Either shot noqueue qdisc, it is even simpler 8)
4192         */
4193        if (dev->flags & IFF_UP) {
4194                int cpu = smp_processor_id(); /* ok because BHs are off */
4195
4196                if (txq->xmit_lock_owner != cpu) {
4197                        if (dev_xmit_recursion())
4198                                goto recursion_alert;
4199
4200                        skb = validate_xmit_skb(skb, dev, &again);
4201                        if (!skb)
4202                                goto out;
4203
4204                        PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
4205                        HARD_TX_LOCK(dev, txq, cpu);
4206
4207                        if (!netif_xmit_stopped(txq)) {
4208                                dev_xmit_recursion_inc();
4209                                skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4210                                dev_xmit_recursion_dec();
4211                                if (dev_xmit_complete(rc)) {
4212                                        HARD_TX_UNLOCK(dev, txq);
4213                                        goto out;
4214                                }
4215                        }
4216                        HARD_TX_UNLOCK(dev, txq);
4217                        net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4218                                             dev->name);
4219                } else {
4220                        /* Recursion is detected! It is possible,
4221                         * unfortunately
4222                         */
4223recursion_alert:
4224                        net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4225                                             dev->name);
4226                }
4227        }
4228
4229        rc = -ENETDOWN;
4230        rcu_read_unlock_bh();
4231
4232        atomic_long_inc(&dev->tx_dropped);
4233        kfree_skb_list(skb);
4234        return rc;
4235out:
4236        rcu_read_unlock_bh();
4237        return rc;
4238}
4239
4240int dev_queue_xmit(struct sk_buff *skb)
4241{
4242        return __dev_queue_xmit(skb, NULL);
4243}
4244EXPORT_SYMBOL(dev_queue_xmit);
4245
4246int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
4247{
4248        return __dev_queue_xmit(skb, sb_dev);
4249}
4250EXPORT_SYMBOL(dev_queue_xmit_accel);
4251
4252int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4253{
4254        struct net_device *dev = skb->dev;
4255        struct sk_buff *orig_skb = skb;
4256        struct netdev_queue *txq;
4257        int ret = NETDEV_TX_BUSY;
4258        bool again = false;
4259
4260        if (unlikely(!netif_running(dev) ||
4261                     !netif_carrier_ok(dev)))
4262                goto drop;
4263
4264        skb = validate_xmit_skb_list(skb, dev, &again);
4265        if (skb != orig_skb)
4266                goto drop;
4267
4268        skb_set_queue_mapping(skb, queue_id);
4269        txq = skb_get_tx_queue(dev, skb);
4270        PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
4271
4272        local_bh_disable();
4273
4274        dev_xmit_recursion_inc();
4275        HARD_TX_LOCK(dev, txq, smp_processor_id());
4276        if (!netif_xmit_frozen_or_drv_stopped(txq))
4277                ret = netdev_start_xmit(skb, dev, txq, false);
4278        HARD_TX_UNLOCK(dev, txq);
4279        dev_xmit_recursion_dec();
4280
4281        local_bh_enable();
4282        return ret;
4283drop:
4284        atomic_long_inc(&dev->tx_dropped);
4285        kfree_skb_list(skb);
4286        return NET_XMIT_DROP;
4287}
4288EXPORT_SYMBOL(__dev_direct_xmit);
4289
4290/*************************************************************************
4291 *                      Receiver routines
4292 *************************************************************************/
4293
4294int netdev_max_backlog __read_mostly = 1000;
4295EXPORT_SYMBOL(netdev_max_backlog);
4296
4297int netdev_tstamp_prequeue __read_mostly = 1;
4298int netdev_budget __read_mostly = 300;
4299/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4300unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4301int weight_p __read_mostly = 64;           /* old backlog weight */
4302int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
4303int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
4304int dev_rx_weight __read_mostly = 64;
4305int dev_tx_weight __read_mostly = 64;
4306/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
4307int gro_normal_batch __read_mostly = 8;
4308
4309/* Called with irq disabled */
4310static inline void ____napi_schedule(struct softnet_data *sd,
4311                                     struct napi_struct *napi)
4312{
4313        struct task_struct *thread;
4314
4315        if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4316                /* Paired with smp_mb__before_atomic() in
4317                 * napi_enable()/dev_set_threaded().
4318                 * Use READ_ONCE() to guarantee a complete
4319                 * read on napi->thread. Only call
4320                 * wake_up_process() when it's not NULL.
4321                 */
4322                thread = READ_ONCE(napi->thread);
4323                if (thread) {
4324                        /* Avoid doing set_bit() if the thread is in
4325                         * INTERRUPTIBLE state, cause napi_thread_wait()
4326                         * makes sure to proceed with napi polling
4327                         * if the thread is explicitly woken from here.
4328                         */
4329                        if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4330                                set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4331                        wake_up_process(thread);
4332                        return;
4333                }
4334        }
4335
4336        list_add_tail(&napi->poll_list, &sd->poll_list);
4337        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4338}
4339
4340#ifdef CONFIG_RPS
4341
4342/* One global table that all flow-based protocols share. */
4343struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4344EXPORT_SYMBOL(rps_sock_flow_table);
4345u32 rps_cpu_mask __read_mostly;
4346EXPORT_SYMBOL(rps_cpu_mask);
4347
4348struct static_key_false rps_needed __read_mostly;
4349EXPORT_SYMBOL(rps_needed);
4350struct static_key_false rfs_needed __read_mostly;
4351EXPORT_SYMBOL(rfs_needed);
4352
4353static struct rps_dev_flow *
4354set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4355            struct rps_dev_flow *rflow, u16 next_cpu)
4356{
4357        if (next_cpu < nr_cpu_ids) {
4358#ifdef CONFIG_RFS_ACCEL
4359                struct netdev_rx_queue *rxqueue;
4360                struct rps_dev_flow_table *flow_table;
4361                struct rps_dev_flow *old_rflow;
4362                u32 flow_id;
4363                u16 rxq_index;
4364                int rc;
4365
4366                /* Should we steer this flow to a different hardware queue? */
4367                if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4368                    !(dev->features & NETIF_F_NTUPLE))
4369                        goto out;
4370                rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4371                if (rxq_index == skb_get_rx_queue(skb))
4372                        goto out;
4373
4374                rxqueue = dev->_rx + rxq_index;
4375                flow_table = rcu_dereference(rxqueue->rps_flow_table);
4376                if (!flow_table)
4377                        goto out;
4378                flow_id = skb_get_hash(skb) & flow_table->mask;
4379                rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4380                                                        rxq_index, flow_id);
4381                if (rc < 0)
4382                        goto out;
4383                old_rflow = rflow;
4384                rflow = &flow_table->flows[flow_id];
4385                rflow->filter = rc;
4386                if (old_rflow->filter == rflow->filter)
4387                        old_rflow->filter = RPS_NO_FILTER;
4388        out:
4389#endif
4390                rflow->last_qtail =
4391                        per_cpu(softnet_data, next_cpu).input_queue_head;
4392        }
4393
4394        rflow->cpu = next_cpu;
4395        return rflow;
4396}
4397
4398/*
4399 * get_rps_cpu is called from netif_receive_skb and returns the target
4400 * CPU from the RPS map of the receiving queue for a given skb.
4401 * rcu_read_lock must be held on entry.
4402 */
4403static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4404                       struct rps_dev_flow **rflowp)
4405{
4406        const struct rps_sock_flow_table *sock_flow_table;
4407        struct netdev_rx_queue *rxqueue = dev->_rx;
4408        struct rps_dev_flow_table *flow_table;
4409        struct rps_map *map;
4410        int cpu = -1;
4411        u32 tcpu;
4412        u32 hash;
4413
4414        if (skb_rx_queue_recorded(skb)) {
4415                u16 index = skb_get_rx_queue(skb);
4416
4417                if (unlikely(index >= dev->real_num_rx_queues)) {
4418                        WARN_ONCE(dev->real_num_rx_queues > 1,
4419                                  "%s received packet on queue %u, but number "
4420                                  "of RX queues is %u\n",
4421                                  dev->name, index, dev->real_num_rx_queues);
4422                        goto done;
4423                }
4424                rxqueue += index;
4425        }
4426
4427        /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4428
4429        flow_table = rcu_dereference(rxqueue->rps_flow_table);
4430        map = rcu_dereference(rxqueue->rps_map);
4431        if (!flow_table && !map)
4432                goto done;
4433
4434        skb_reset_network_header(skb);
4435        hash = skb_get_hash(skb);
4436        if (!hash)
4437                goto done;
4438
4439        sock_flow_table = rcu_dereference(rps_sock_flow_table);
4440        if (flow_table && sock_flow_table) {
4441                struct rps_dev_flow *rflow;
4442                u32 next_cpu;
4443                u32 ident;
4444
4445                /* First check into global flow table if there is a match */
4446                ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4447                if ((ident ^ hash) & ~rps_cpu_mask)
4448                        goto try_rps;
4449
4450                next_cpu = ident & rps_cpu_mask;
4451
4452                /* OK, now we know there is a match,
4453                 * we can look at the local (per receive queue) flow table
4454                 */
4455                rflow = &flow_table->flows[hash & flow_table->mask];
4456                tcpu = rflow->cpu;
4457
4458                /*
4459                 * If the desired CPU (where last recvmsg was done) is
4460                 * different from current CPU (one in the rx-queue flow
4461                 * table entry), switch if one of the following holds:
4462                 *   - Current CPU is unset (>= nr_cpu_ids).
4463                 *   - Current CPU is offline.
4464                 *   - The current CPU's queue tail has advanced beyond the
4465                 *     last packet that was enqueued using this table entry.
4466                 *     This guarantees that all previous packets for the flow
4467                 *     have been dequeued, thus preserving in order delivery.
4468                 */
4469                if (unlikely(tcpu != next_cpu) &&
4470                    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4471                     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4472                      rflow->last_qtail)) >= 0)) {
4473                        tcpu = next_cpu;
4474                        rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4475                }
4476
4477                if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4478                        *rflowp = rflow;
4479                        cpu = tcpu;
4480                        goto done;
4481                }
4482        }
4483
4484try_rps:
4485
4486        if (map) {
4487                tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4488                if (cpu_online(tcpu)) {
4489                        cpu = tcpu;
4490                        goto done;
4491                }
4492        }
4493
4494done:
4495        return cpu;
4496}
4497
4498#ifdef CONFIG_RFS_ACCEL
4499
4500/**
4501 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4502 * @dev: Device on which the filter was set
4503 * @rxq_index: RX queue index
4504 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4505 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4506 *
4507 * Drivers that implement ndo_rx_flow_steer() should periodically call
4508 * this function for each installed filter and remove the filters for
4509 * which it returns %true.
4510 */
4511bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4512                         u32 flow_id, u16 filter_id)
4513{
4514        struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4515        struct rps_dev_flow_table *flow_table;
4516        struct rps_dev_flow *rflow;
4517        bool expire = true;
4518        unsigned int cpu;
4519
4520        rcu_read_lock();
4521        flow_table = rcu_dereference(rxqueue->rps_flow_table);
4522        if (flow_table && flow_id <= flow_table->mask) {
4523                rflow = &flow_table->flows[flow_id];
4524                cpu = READ_ONCE(rflow->cpu);
4525                if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4526                    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4527                           rflow->last_qtail) <
4528                     (int)(10 * flow_table->mask)))
4529                        expire = false;
4530        }
4531        rcu_read_unlock();
4532        return expire;
4533}
4534EXPORT_SYMBOL(rps_may_expire_flow);
4535
4536#endif /* CONFIG_RFS_ACCEL */
4537
4538/* Called from hardirq (IPI) context */
4539static void rps_trigger_softirq(void *data)
4540{
4541        struct softnet_data *sd = data;
4542
4543        ____napi_schedule(sd, &sd->backlog);
4544        sd->received_rps++;
4545}
4546
4547#endif /* CONFIG_RPS */
4548
4549/*
4550 * Check if this softnet_data structure is another cpu one
4551 * If yes, queue it to our IPI list and return 1
4552 * If no, return 0
4553 */
4554static int rps_ipi_queued(struct softnet_data *sd)
4555{
4556#ifdef CONFIG_RPS
4557        struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4558
4559        if (sd != mysd) {
4560                sd->rps_ipi_next = mysd->rps_ipi_list;
4561                mysd->rps_ipi_list = sd;
4562
4563                __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4564                return 1;
4565        }
4566#endif /* CONFIG_RPS */
4567        return 0;
4568}
4569
4570#ifdef CONFIG_NET_FLOW_LIMIT
4571int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4572#endif
4573
4574static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4575{
4576#ifdef CONFIG_NET_FLOW_LIMIT
4577        struct sd_flow_limit *fl;
4578        struct softnet_data *sd;
4579        unsigned int old_flow, new_flow;
4580
4581        if (qlen < (netdev_max_backlog >> 1))
4582                return false;
4583
4584        sd = this_cpu_ptr(&softnet_data);
4585
4586        rcu_read_lock();
4587        fl = rcu_dereference(sd->flow_limit);
4588        if (fl) {
4589                new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4590                old_flow = fl->history[fl->history_head];
4591                fl->history[fl->history_head] = new_flow;
4592
4593                fl->history_head++;
4594                fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4595
4596                if (likely(fl->buckets[old_flow]))
4597                        fl->buckets[old_flow]--;
4598
4599                if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4600                        fl->count++;
4601                        rcu_read_unlock();
4602                        return true;
4603                }
4604        }
4605        rcu_read_unlock();
4606#endif
4607        return false;
4608}
4609
4610/*
4611 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4612 * queue (may be a remote CPU queue).
4613 */
4614static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4615                              unsigned int *qtail)
4616{
4617        struct softnet_data *sd;
4618        unsigned long flags;
4619        unsigned int qlen;
4620
4621        sd = &per_cpu(softnet_data, cpu);
4622
4623        local_irq_save(flags);
4624
4625        rps_lock(sd);
4626        if (!netif_running(skb->dev))
4627                goto drop;
4628        qlen = skb_queue_len(&sd->input_pkt_queue);
4629        if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4630                if (qlen) {
4631enqueue:
4632                        __skb_queue_tail(&sd->input_pkt_queue, skb);
4633                        input_queue_tail_incr_save(sd, qtail);
4634                        rps_unlock(sd);
4635                        local_irq_restore(flags);
4636                        return NET_RX_SUCCESS;
4637                }
4638
4639                /* Schedule NAPI for backlog device
4640                 * We can use non atomic operation since we own the queue lock
4641                 */
4642                if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
4643                        if (!rps_ipi_queued(sd))
4644                                ____napi_schedule(sd, &sd->backlog);
4645                }
4646                goto enqueue;
4647        }
4648
4649drop:
4650        sd->dropped++;
4651        rps_unlock(sd);
4652
4653        local_irq_restore(flags);
4654
4655        atomic_long_inc(&skb->dev->rx_dropped);
4656        kfree_skb(skb);
4657        return NET_RX_DROP;
4658}
4659
4660static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4661{
4662        struct net_device *dev = skb->dev;
4663        struct netdev_rx_queue *rxqueue;
4664
4665        rxqueue = dev->_rx;
4666
4667        if (skb_rx_queue_recorded(skb)) {
4668                u16 index = skb_get_rx_queue(skb);
4669
4670                if (unlikely(index >= dev->real_num_rx_queues)) {
4671                        WARN_ONCE(dev->real_num_rx_queues > 1,
4672                                  "%s received packet on queue %u, but number "
4673                                  "of RX queues is %u\n",
4674                                  dev->name, index, dev->real_num_rx_queues);
4675
4676                        return rxqueue; /* Return first rxqueue */
4677                }
4678                rxqueue += index;
4679        }
4680        return rxqueue;
4681}
4682
4683u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4684                             struct bpf_prog *xdp_prog)
4685{
4686        void *orig_data, *orig_data_end, *hard_start;
4687        struct netdev_rx_queue *rxqueue;
4688        bool orig_bcast, orig_host;
4689        u32 mac_len, frame_sz;
4690        __be16 orig_eth_type;
4691        struct ethhdr *eth;
4692        u32 metalen, act;
4693        int off;
4694
4695        /* The XDP program wants to see the packet starting at the MAC
4696         * header.
4697         */
4698        mac_len = skb->data - skb_mac_header(skb);
4699        hard_start = skb->data - skb_headroom(skb);
4700
4701        /* SKB "head" area always have tailroom for skb_shared_info */
4702        frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4703        frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4704
4705        rxqueue = netif_get_rxqueue(skb);
4706        xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4707        xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4708                         skb_headlen(skb) + mac_len, true);
4709
4710        orig_data_end = xdp->data_end;
4711        orig_data = xdp->data;
4712        eth = (struct ethhdr *)xdp->data;
4713        orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4714        orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4715        orig_eth_type = eth->h_proto;
4716
4717        act = bpf_prog_run_xdp(xdp_prog, xdp);
4718
4719        /* check if bpf_xdp_adjust_head was used */
4720        off = xdp->data - orig_data;
4721        if (off) {
4722                if (off > 0)
4723                        __skb_pull(skb, off);
4724                else if (off < 0)
4725                        __skb_push(skb, -off);
4726
4727                skb->mac_header += off;
4728                skb_reset_network_header(skb);
4729        }
4730
4731        /* check if bpf_xdp_adjust_tail was used */
4732        off = xdp->data_end - orig_data_end;
4733        if (off != 0) {
4734                skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4735                skb->len += off; /* positive on grow, negative on shrink */
4736        }
4737
4738        /* check if XDP changed eth hdr such SKB needs update */
4739        eth = (struct ethhdr *)xdp->data;
4740        if ((orig_eth_type != eth->h_proto) ||
4741            (orig_host != ether_addr_equal_64bits(eth->h_dest,
4742                                                  skb->dev->dev_addr)) ||
4743            (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4744                __skb_push(skb, ETH_HLEN);
4745                skb->pkt_type = PACKET_HOST;
4746                skb->protocol = eth_type_trans(skb, skb->dev);
4747        }
4748
4749        /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4750         * before calling us again on redirect path. We do not call do_redirect
4751         * as we leave that up to the caller.
4752         *
4753         * Caller is responsible for managing lifetime of skb (i.e. calling
4754         * kfree_skb in response to actions it cannot handle/XDP_DROP).
4755         */
4756        switch (act) {
4757        case XDP_REDIRECT:
4758        case XDP_TX:
4759                __skb_push(skb, mac_len);
4760                break;
4761        case XDP_PASS:
4762                metalen = xdp->data - xdp->data_meta;
4763                if (metalen)
4764                        skb_metadata_set(skb, metalen);
4765                break;
4766        }
4767
4768        return act;
4769}
4770
4771static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4772                                     struct xdp_buff *xdp,
4773                                     struct bpf_prog *xdp_prog)
4774{
4775        u32 act = XDP_DROP;
4776
4777        /* Reinjected packets coming from act_mirred or similar should
4778         * not get XDP generic processing.
4779         */
4780        if (skb_is_redirected(skb))
4781                return XDP_PASS;
4782
4783        /* XDP packets must be linear and must have sufficient headroom
4784         * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4785         * native XDP provides, thus we need to do it here as well.
4786         */
4787        if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4788            skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4789                int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4790                int troom = skb->tail + skb->data_len - skb->end;
4791
4792                /* In case we have to go down the path and also linearize,
4793                 * then lets do the pskb_expand_head() work just once here.
4794                 */
4795                if (pskb_expand_head(skb,
4796                                     hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4797                                     troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4798                        goto do_drop;
4799                if (skb_linearize(skb))
4800                        goto do_drop;
4801        }
4802
4803        act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4804        switch (act) {
4805        case XDP_REDIRECT:
4806        case XDP_TX:
4807        case XDP_PASS:
4808                break;
4809        default:
4810                bpf_warn_invalid_xdp_action(act);
4811                fallthrough;
4812        case XDP_ABORTED:
4813                trace_xdp_exception(skb->dev, xdp_prog, act);
4814                fallthrough;
4815        case XDP_DROP:
4816        do_drop:
4817                kfree_skb(skb);
4818                break;
4819        }
4820
4821        return act;
4822}
4823
4824/* When doing generic XDP we have to bypass the qdisc layer and the
4825 * network taps in order to match in-driver-XDP behavior.
4826 */
4827void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4828{
4829        struct net_device *dev = skb->dev;
4830        struct netdev_queue *txq;
4831        bool free_skb = true;
4832        int cpu, rc;
4833
4834        txq = netdev_core_pick_tx(dev, skb, NULL);
4835        cpu = smp_processor_id();
4836        HARD_TX_LOCK(dev, txq, cpu);
4837        if (!netif_xmit_stopped(txq)) {
4838                rc = netdev_start_xmit(skb, dev, txq, 0);
4839                if (dev_xmit_complete(rc))
4840                        free_skb = false;
4841        }
4842        HARD_TX_UNLOCK(dev, txq);
4843        if (free_skb) {
4844                trace_xdp_exception(dev, xdp_prog, XDP_TX);
4845                kfree_skb(skb);
4846        }
4847}
4848
4849static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4850
4851int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4852{
4853        if (xdp_prog) {
4854                struct xdp_buff xdp;
4855                u32 act;
4856                int err;
4857
4858                act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4859                if (act != XDP_PASS) {
4860                        switch (act) {
4861                        case XDP_REDIRECT:
4862                                err = xdp_do_generic_redirect(skb->dev, skb,
4863                                                              &xdp, xdp_prog);
4864                                if (err)
4865                                        goto out_redir;
4866                                break;
4867                        case XDP_TX:
4868                                generic_xdp_tx(skb, xdp_prog);
4869                                break;
4870                        }
4871                        return XDP_DROP;
4872                }
4873        }
4874        return XDP_PASS;
4875out_redir:
4876        kfree_skb(skb);
4877        return XDP_DROP;
4878}
4879EXPORT_SYMBOL_GPL(do_xdp_generic);
4880
4881static int netif_rx_internal(struct sk_buff *skb)
4882{
4883        int ret;
4884
4885        net_timestamp_check(netdev_tstamp_prequeue, skb);
4886
4887        trace_netif_rx(skb);
4888
4889#ifdef CONFIG_RPS
4890        if (static_branch_unlikely(&rps_needed)) {
4891                struct rps_dev_flow voidflow, *rflow = &voidflow;
4892                int cpu;
4893
4894                preempt_disable();
4895                rcu_read_lock();
4896
4897                cpu = get_rps_cpu(skb->dev, skb, &rflow);
4898                if (cpu < 0)
4899                        cpu = smp_processor_id();
4900
4901                ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4902
4903                rcu_read_unlock();
4904                preempt_enable();
4905        } else
4906#endif
4907        {
4908                unsigned int qtail;
4909
4910                ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4911                put_cpu();
4912        }
4913        return ret;
4914}
4915
4916/**
4917 *      netif_rx        -       post buffer to the network code
4918 *      @skb: buffer to post
4919 *
4920 *      This function receives a packet from a device driver and queues it for
4921 *      the upper (protocol) levels to process.  It always succeeds. The buffer
4922 *      may be dropped during processing for congestion control or by the
4923 *      protocol layers.
4924 *
4925 *      return values:
4926 *      NET_RX_SUCCESS  (no congestion)
4927 *      NET_RX_DROP     (packet was dropped)
4928 *
4929 */
4930
4931int netif_rx(struct sk_buff *skb)
4932{
4933        int ret;
4934
4935        trace_netif_rx_entry(skb);
4936
4937        ret = netif_rx_internal(skb);
4938        trace_netif_rx_exit(ret);
4939
4940        return ret;
4941}
4942EXPORT_SYMBOL(netif_rx);
4943
4944int netif_rx_ni(struct sk_buff *skb)
4945{
4946        int err;
4947
4948        trace_netif_rx_ni_entry(skb);
4949
4950        preempt_disable();
4951        err = netif_rx_internal(skb);
4952        if (local_softirq_pending())
4953                do_softirq();
4954        preempt_enable();
4955        trace_netif_rx_ni_exit(err);
4956
4957        return err;
4958}
4959EXPORT_SYMBOL(netif_rx_ni);
4960
4961int netif_rx_any_context(struct sk_buff *skb)
4962{
4963        /*
4964         * If invoked from contexts which do not invoke bottom half
4965         * processing either at return from interrupt or when softrqs are
4966         * reenabled, use netif_rx_ni() which invokes bottomhalf processing
4967         * directly.
4968         */
4969        if (in_interrupt())
4970                return netif_rx(skb);
4971        else
4972                return netif_rx_ni(skb);
4973}
4974EXPORT_SYMBOL(netif_rx_any_context);
4975
4976static __latent_entropy void net_tx_action(struct softirq_action *h)
4977{
4978        struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4979
4980        if (sd->completion_queue) {
4981                struct sk_buff *clist;
4982
4983                local_irq_disable();
4984                clist = sd->completion_queue;
4985                sd->completion_queue = NULL;
4986                local_irq_enable();
4987
4988                while (clist) {
4989                        struct sk_buff *skb = clist;
4990
4991                        clist = clist->next;
4992
4993                        WARN_ON(refcount_read(&skb->users));
4994                        if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4995                                trace_consume_skb(skb);
4996                        else
4997                                trace_kfree_skb(skb, net_tx_action);
4998
4999                        if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5000                                __kfree_skb(skb);
5001                        else
5002                                __kfree_skb_defer(skb);
5003                }
5004        }
5005
5006        if (sd->output_queue) {
5007                struct Qdisc *head;
5008
5009                local_irq_disable();
5010                head = sd->output_queue;
5011                sd->output_queue = NULL;
5012                sd->output_queue_tailp = &sd->output_queue;
5013                local_irq_enable();
5014
5015                rcu_read_lock();
5016
5017                while (head) {
5018                        struct Qdisc *q = head;
5019                        spinlock_t *root_lock = NULL;
5020
5021                        head = head->next_sched;
5022
5023                        /* We need to make sure head->next_sched is read
5024                         * before clearing __QDISC_STATE_SCHED
5025                         */
5026                        smp_mb__before_atomic();
5027
5028                        if (!(q->flags & TCQ_F_NOLOCK)) {
5029                                root_lock = qdisc_lock(q);
5030                                spin_lock(root_lock);
5031                        } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5032                                                     &q->state))) {
5033                                /* There is a synchronize_net() between
5034                                 * STATE_DEACTIVATED flag being set and
5035                                 * qdisc_reset()/some_qdisc_is_busy() in
5036                                 * dev_deactivate(), so we can safely bail out
5037                                 * early here to avoid data race between
5038                                 * qdisc_deactivate() and some_qdisc_is_busy()
5039                                 * for lockless qdisc.
5040                                 */
5041                                clear_bit(__QDISC_STATE_SCHED, &q->state);
5042                                continue;
5043                        }
5044
5045                        clear_bit(__QDISC_STATE_SCHED, &q->state);
5046                        qdisc_run(q);
5047                        if (root_lock)
5048                                spin_unlock(root_lock);
5049                }
5050
5051                rcu_read_unlock();
5052        }
5053
5054        xfrm_dev_backlog(sd);
5055}
5056
5057#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5058/* This hook is defined here for ATM LANE */
5059int (*br_fdb_test_addr_hook)(struct net_device *dev,
5060                             unsigned char *addr) __read_mostly;
5061EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5062#endif
5063
5064static inline struct sk_buff *
5065sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5066                   struct net_device *orig_dev, bool *another)
5067{
5068#ifdef CONFIG_NET_CLS_ACT
5069        struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5070        struct tcf_result cl_res;
5071
5072        /* If there's at least one ingress present somewhere (so
5073         * we get here via enabled static key), remaining devices
5074         * that are not configured with an ingress qdisc will bail
5075         * out here.
5076         */
5077        if (!miniq)
5078                return skb;
5079
5080        if (*pt_prev) {
5081                *ret = deliver_skb(skb, *pt_prev, orig_dev);
5082                *pt_prev = NULL;
5083        }
5084
5085        qdisc_skb_cb(skb)->pkt_len = skb->len;
5086        qdisc_skb_cb(skb)->mru = 0;
5087        qdisc_skb_cb(skb)->post_ct = false;
5088        skb->tc_at_ingress = 1;
5089        mini_qdisc_bstats_cpu_update(miniq, skb);
5090
5091        switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
5092        case TC_ACT_OK:
5093        case TC_ACT_RECLASSIFY:
5094                skb->tc_index = TC_H_MIN(cl_res.classid);
5095                break;
5096        case TC_ACT_SHOT:
5097                mini_qdisc_qstats_cpu_drop(miniq);
5098                kfree_skb(skb);
5099                return NULL;
5100        case TC_ACT_STOLEN:
5101        case TC_ACT_QUEUED:
5102        case TC_ACT_TRAP:
5103                consume_skb(skb);
5104                return NULL;
5105        case TC_ACT_REDIRECT:
5106                /* skb_mac_header check was done by cls/act_bpf, so
5107                 * we can safely push the L2 header back before
5108                 * redirecting to another netdev
5109                 */
5110                __skb_push(skb, skb->mac_len);
5111                if (skb_do_redirect(skb) == -EAGAIN) {
5112                        __skb_pull(skb, skb->mac_len);
5113                        *another = true;
5114                        break;
5115                }
5116                return NULL;
5117        case TC_ACT_CONSUMED:
5118                return NULL;
5119        default:
5120                break;
5121        }
5122#endif /* CONFIG_NET_CLS_ACT */
5123        return skb;
5124}
5125
5126/**
5127 *      netdev_is_rx_handler_busy - check if receive handler is registered
5128 *      @dev: device to check
5129 *
5130 *      Check if a receive handler is already registered for a given device.
5131 *      Return true if there one.
5132 *
5133 *      The caller must hold the rtnl_mutex.
5134 */
5135bool netdev_is_rx_handler_busy(struct net_device *dev)
5136{
5137        ASSERT_RTNL();
5138        return dev && rtnl_dereference(dev->rx_handler);
5139}
5140EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5141
5142/**
5143 *      netdev_rx_handler_register - register receive handler
5144 *      @dev: device to register a handler for
5145 *      @rx_handler: receive handler to register
5146 *      @rx_handler_data: data pointer that is used by rx handler
5147 *
5148 *      Register a receive handler for a device. This handler will then be
5149 *      called from __netif_receive_skb. A negative errno code is returned
5150 *      on a failure.
5151 *
5152 *      The caller must hold the rtnl_mutex.
5153 *
5154 *      For a general description of rx_handler, see enum rx_handler_result.
5155 */
5156int netdev_rx_handler_register(struct net_device *dev,
5157                               rx_handler_func_t *rx_handler,
5158                               void *rx_handler_data)
5159{
5160        if (netdev_is_rx_handler_busy(dev))
5161                return -EBUSY;
5162
5163        if (dev->priv_flags & IFF_NO_RX_HANDLER)
5164                return -EINVAL;
5165
5166        /* Note: rx_handler_data must be set before rx_handler */
5167        rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5168        rcu_assign_pointer(dev->rx_handler, rx_handler);
5169
5170        return 0;
5171}
5172EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5173
5174/**
5175 *      netdev_rx_handler_unregister - unregister receive handler
5176 *      @dev: device to unregister a handler from
5177 *
5178 *      Unregister a receive handler from a device.
5179 *
5180 *      The caller must hold the rtnl_mutex.
5181 */
5182void netdev_rx_handler_unregister(struct net_device *dev)
5183{
5184
5185        ASSERT_RTNL();
5186        RCU_INIT_POINTER(dev->rx_handler, NULL);
5187        /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5188         * section has a guarantee to see a non NULL rx_handler_data
5189         * as well.
5190         */
5191        synchronize_net();
5192        RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5193}
5194EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5195
5196/*
5197 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5198 * the special handling of PFMEMALLOC skbs.
5199 */
5200static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5201{
5202        switch (skb->protocol) {
5203        case htons(ETH_P_ARP):
5204        case htons(ETH_P_IP):
5205        case htons(ETH_P_IPV6):
5206        case htons(ETH_P_8021Q):
5207        case htons(ETH_P_8021AD):
5208                return true;
5209        default:
5210                return false;
5211        }
5212}
5213
5214static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5215                             int *ret, struct net_device *orig_dev)
5216{
5217        if (nf_hook_ingress_active(skb)) {
5218                int ingress_retval;
5219
5220                if (*pt_prev) {
5221                        *ret = deliver_skb(skb, *pt_prev, orig_dev);
5222                        *pt_prev = NULL;
5223                }
5224
5225                rcu_read_lock();
5226                ingress_retval = nf_hook_ingress(skb);
5227                rcu_read_unlock();
5228                return ingress_retval;
5229        }
5230        return 0;
5231}
5232
5233static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5234                                    struct packet_type **ppt_prev)
5235{
5236        struct packet_type *ptype, *pt_prev;
5237        rx_handler_func_t *rx_handler;
5238        struct sk_buff *skb = *pskb;
5239        struct net_device *orig_dev;
5240        bool deliver_exact = false;
5241        int ret = NET_RX_DROP;
5242        __be16 type;
5243
5244        net_timestamp_check(!netdev_tstamp_prequeue, skb);
5245
5246        trace_netif_receive_skb(skb);
5247
5248        orig_dev = skb->dev;
5249
5250        skb_reset_network_header(skb);
5251        if (!skb_transport_header_was_set(skb))
5252                skb_reset_transport_header(skb);
5253        skb_reset_mac_len(skb);
5254
5255        pt_prev = NULL;
5256
5257another_round:
5258        skb->skb_iif = skb->dev->ifindex;
5259
5260        __this_cpu_inc(softnet_data.processed);
5261
5262        if (static_branch_unlikely(&generic_xdp_needed_key)) {
5263                int ret2;
5264
5265                migrate_disable();
5266                ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5267                migrate_enable();
5268
5269                if (ret2 != XDP_PASS) {
5270                        ret = NET_RX_DROP;
5271                        goto out;
5272                }
5273        }
5274
5275        if (eth_type_vlan(skb->protocol)) {
5276                skb = skb_vlan_untag(skb);
5277                if (unlikely(!skb))
5278                        goto out;
5279        }
5280
5281        if (skb_skip_tc_classify(skb))
5282                goto skip_classify;
5283
5284        if (pfmemalloc)
5285                goto skip_taps;
5286
5287        list_for_each_entry_rcu(ptype, &ptype_all, list) {
5288                if (pt_prev)
5289                        ret = deliver_skb(skb, pt_prev, orig_dev);
5290                pt_prev = ptype;
5291        }
5292
5293        list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5294                if (pt_prev)
5295                        ret = deliver_skb(skb, pt_prev, orig_dev);
5296                pt_prev = ptype;
5297        }
5298
5299skip_taps:
5300#ifdef CONFIG_NET_INGRESS
5301        if (static_branch_unlikely(&ingress_needed_key)) {
5302                bool another = false;
5303
5304                skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5305                                         &another);
5306                if (another)
5307                        goto another_round;
5308                if (!skb)
5309                        goto out;
5310
5311                if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5312                        goto out;
5313        }
5314#endif
5315        skb_reset_redirect(skb);
5316skip_classify:
5317        if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5318                goto drop;
5319
5320        if (skb_vlan_tag_present(skb)) {
5321                if (pt_prev) {
5322                        ret = deliver_skb(skb, pt_prev, orig_dev);
5323                        pt_prev = NULL;
5324                }
5325                if (vlan_do_receive(&skb))
5326                        goto another_round;
5327                else if (unlikely(!skb))
5328                        goto out;
5329        }
5330
5331        rx_handler = rcu_dereference(skb->dev->rx_handler);
5332        if (rx_handler) {
5333                if (pt_prev) {
5334                        ret = deliver_skb(skb, pt_prev, orig_dev);
5335                        pt_prev = NULL;
5336                }
5337                switch (rx_handler(&skb)) {
5338                case RX_HANDLER_CONSUMED:
5339                        ret = NET_RX_SUCCESS;
5340                        goto out;
5341                case RX_HANDLER_ANOTHER:
5342                        goto another_round;
5343                case RX_HANDLER_EXACT:
5344                        deliver_exact = true;
5345                        break;
5346                case RX_HANDLER_PASS:
5347                        break;
5348                default:
5349                        BUG();
5350                }
5351        }
5352
5353        if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5354check_vlan_id:
5355                if (skb_vlan_tag_get_id(skb)) {
5356                        /* Vlan id is non 0 and vlan_do_receive() above couldn't
5357                         * find vlan device.
5358                         */
5359                        skb->pkt_type = PACKET_OTHERHOST;
5360                } else if (eth_type_vlan(skb->protocol)) {
5361                        /* Outer header is 802.1P with vlan 0, inner header is
5362                         * 802.1Q or 802.1AD and vlan_do_receive() above could
5363                         * not find vlan dev for vlan id 0.
5364                         */
5365                        __vlan_hwaccel_clear_tag(skb);
5366                        skb = skb_vlan_untag(skb);
5367                        if (unlikely(!skb))
5368                                goto out;
5369                        if (vlan_do_receive(&skb))
5370                                /* After stripping off 802.1P header with vlan 0
5371                                 * vlan dev is found for inner header.
5372                                 */
5373                                goto another_round;
5374                        else if (unlikely(!skb))
5375                                goto out;
5376                        else
5377                                /* We have stripped outer 802.1P vlan 0 header.
5378                                 * But could not find vlan dev.
5379                                 * check again for vlan id to set OTHERHOST.
5380                                 */
5381                                goto check_vlan_id;
5382                }
5383                /* Note: we might in the future use prio bits
5384                 * and set skb->priority like in vlan_do_receive()
5385                 * For the time being, just ignore Priority Code Point
5386                 */
5387                __vlan_hwaccel_clear_tag(skb);
5388        }
5389
5390        type = skb->protocol;
5391
5392        /* deliver only exact match when indicated */
5393        if (likely(!deliver_exact)) {
5394                deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5395                                       &ptype_base[ntohs(type) &
5396                                                   PTYPE_HASH_MASK]);
5397        }
5398
5399        deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5400                               &orig_dev->ptype_specific);
5401
5402        if (unlikely(skb->dev != orig_dev)) {
5403                deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5404                                       &skb->dev->ptype_specific);
5405        }
5406
5407        if (pt_prev) {
5408                if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5409                        goto drop;
5410                *ppt_prev = pt_prev;
5411        } else {
5412drop:
5413                if (!deliver_exact)
5414                        atomic_long_inc(&skb->dev->rx_dropped);
5415                else
5416                        atomic_long_inc(&skb->dev->rx_nohandler);
5417                kfree_skb(skb);
5418                /* Jamal, now you will not able to escape explaining
5419                 * me how you were going to use this. :-)
5420                 */
5421                ret = NET_RX_DROP;
5422        }
5423
5424out:
5425        /* The invariant here is that if *ppt_prev is not NULL
5426         * then skb should also be non-NULL.
5427         *
5428         * Apparently *ppt_prev assignment above holds this invariant due to
5429         * skb dereferencing near it.
5430         */
5431        *pskb = skb;
5432        return ret;
5433}
5434
5435static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5436{
5437        struct net_device *orig_dev = skb->dev;
5438        struct packet_type *pt_prev = NULL;
5439        int ret;
5440
5441        ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5442        if (pt_prev)
5443                ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5444                                         skb->dev, pt_prev, orig_dev);
5445        return ret;
5446}
5447
5448/**
5449 *      netif_receive_skb_core - special purpose version of netif_receive_skb
5450 *      @skb: buffer to process
5451 *
5452 *      More direct receive version of netif_receive_skb().  It should
5453 *      only be used by callers that have a need to skip RPS and Generic XDP.
5454 *      Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5455 *
5456 *      This function may only be called from softirq context and interrupts
5457 *      should be enabled.
5458 *
5459 *      Return values (usually ignored):
5460 *      NET_RX_SUCCESS: no congestion
5461 *      NET_RX_DROP: packet was dropped
5462 */
5463int netif_receive_skb_core(struct sk_buff *skb)
5464{
5465        int ret;
5466
5467        rcu_read_lock();
5468        ret = __netif_receive_skb_one_core(skb, false);
5469        rcu_read_unlock();
5470
5471        return ret;
5472}
5473EXPORT_SYMBOL(netif_receive_skb_core);
5474
5475static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5476                                                  struct packet_type *pt_prev,
5477                                                  struct net_device *orig_dev)
5478{
5479        struct sk_buff *skb, *next;
5480
5481        if (!pt_prev)
5482                return;
5483        if (list_empty(head))
5484                return;
5485        if (pt_prev->list_func != NULL)
5486                INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5487                                   ip_list_rcv, head, pt_prev, orig_dev);
5488        else
5489                list_for_each_entry_safe(skb, next, head, list) {
5490                        skb_list_del_init(skb);
5491                        pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5492                }
5493}
5494
5495static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5496{
5497        /* Fast-path assumptions:
5498         * - There is no RX handler.
5499         * - Only one packet_type matches.
5500         * If either of these fails, we will end up doing some per-packet
5501         * processing in-line, then handling the 'last ptype' for the whole
5502         * sublist.  This can't cause out-of-order delivery to any single ptype,
5503         * because the 'last ptype' must be constant across the sublist, and all
5504         * other ptypes are handled per-packet.
5505         */
5506        /* Current (common) ptype of sublist */
5507        struct packet_type *pt_curr = NULL;
5508        /* Current (common) orig_dev of sublist */
5509        struct net_device *od_curr = NULL;
5510        struct list_head sublist;
5511        struct sk_buff *skb, *next;
5512
5513        INIT_LIST_HEAD(&sublist);
5514        list_for_each_entry_safe(skb, next, head, list) {
5515                struct net_device *orig_dev = skb->dev;
5516                struct packet_type *pt_prev = NULL;
5517
5518                skb_list_del_init(skb);
5519                __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5520                if (!pt_prev)
5521                        continue;
5522                if (pt_curr != pt_prev || od_curr != orig_dev) {
5523                        /* dispatch old sublist */
5524                        __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5525                        /* start new sublist */
5526                        INIT_LIST_HEAD(&sublist);
5527                        pt_curr = pt_prev;
5528                        od_curr = orig_dev;
5529                }
5530                list_add_tail(&skb->list, &sublist);
5531        }
5532
5533        /* dispatch final sublist */
5534        __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5535}
5536
5537static int __netif_receive_skb(struct sk_buff *skb)
5538{
5539        int ret;
5540
5541        if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5542                unsigned int noreclaim_flag;
5543
5544                /*
5545                 * PFMEMALLOC skbs are special, they should
5546                 * - be delivered to SOCK_MEMALLOC sockets only
5547                 * - stay away from userspace
5548                 * - have bounded memory usage
5549                 *
5550                 * Use PF_MEMALLOC as this saves us from propagating the allocation
5551                 * context down to all allocation sites.
5552                 */
5553                noreclaim_flag = memalloc_noreclaim_save();
5554                ret = __netif_receive_skb_one_core(skb, true);
5555                memalloc_noreclaim_restore(noreclaim_flag);
5556        } else
5557                ret = __netif_receive_skb_one_core(skb, false);
5558
5559        return ret;
5560}
5561
5562static void __netif_receive_skb_list(struct list_head *head)
5563{
5564        unsigned long noreclaim_flag = 0;
5565        struct sk_buff *skb, *next;
5566        bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5567
5568        list_for_each_entry_safe(skb, next, head, list) {
5569                if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5570                        struct list_head sublist;
5571
5572                        /* Handle the previous sublist */
5573                        list_cut_before(&sublist, head, &skb->list);
5574                        if (!list_empty(&sublist))
5575                                __netif_receive_skb_list_core(&sublist, pfmemalloc);
5576                        pfmemalloc = !pfmemalloc;
5577                        /* See comments in __netif_receive_skb */
5578                        if (pfmemalloc)
5579                                noreclaim_flag = memalloc_noreclaim_save();
5580                        else
5581                                memalloc_noreclaim_restore(noreclaim_flag);
5582                }
5583        }
5584        /* Handle the remaining sublist */
5585        if (!list_empty(head))
5586                __netif_receive_skb_list_core(head, pfmemalloc);
5587        /* Restore pflags */
5588        if (pfmemalloc)
5589                memalloc_noreclaim_restore(noreclaim_flag);
5590}
5591
5592static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5593{
5594        struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5595        struct bpf_prog *new = xdp->prog;
5596        int ret = 0;
5597
5598        switch (xdp->command) {
5599        case XDP_SETUP_PROG:
5600                rcu_assign_pointer(dev->xdp_prog, new);
5601                if (old)
5602                        bpf_prog_put(old);
5603
5604                if (old && !new) {
5605                        static_branch_dec(&generic_xdp_needed_key);
5606                } else if (new && !old) {
5607                        static_branch_inc(&generic_xdp_needed_key);
5608                        dev_disable_lro(dev);
5609                        dev_disable_gro_hw(dev);
5610                }
5611                break;
5612
5613        default:
5614                ret = -EINVAL;
5615                break;
5616        }
5617
5618        return ret;
5619}
5620
5621static int netif_receive_skb_internal(struct sk_buff *skb)
5622{
5623        int ret;
5624
5625        net_timestamp_check(netdev_tstamp_prequeue, skb);
5626
5627        if (skb_defer_rx_timestamp(skb))
5628                return NET_RX_SUCCESS;
5629
5630        rcu_read_lock();
5631#ifdef CONFIG_RPS
5632        if (static_branch_unlikely(&rps_needed)) {
5633                struct rps_dev_flow voidflow, *rflow = &voidflow;
5634                int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5635
5636                if (cpu >= 0) {
5637                        ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5638                        rcu_read_unlock();
5639                        return ret;
5640                }
5641        }
5642#endif
5643        ret = __netif_receive_skb(skb);
5644        rcu_read_unlock();
5645        return ret;
5646}
5647
5648static void netif_receive_skb_list_internal(struct list_head *head)
5649{
5650        struct sk_buff *skb, *next;
5651        struct list_head sublist;
5652
5653        INIT_LIST_HEAD(&sublist);
5654        list_for_each_entry_safe(skb, next, head, list) {
5655                net_timestamp_check(netdev_tstamp_prequeue, skb);
5656                skb_list_del_init(skb);
5657                if (!skb_defer_rx_timestamp(skb))
5658                        list_add_tail(&skb->list, &sublist);
5659        }
5660        list_splice_init(&sublist, head);
5661
5662        rcu_read_lock();
5663#ifdef CONFIG_RPS
5664        if (static_branch_unlikely(&rps_needed)) {
5665                list_for_each_entry_safe(skb, next, head, list) {
5666                        struct rps_dev_flow voidflow, *rflow = &voidflow;
5667                        int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5668
5669                        if (cpu >= 0) {
5670                                /* Will be handled, remove from list */
5671                                skb_list_del_init(skb);
5672                                enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5673                        }
5674                }
5675        }
5676#endif
5677        __netif_receive_skb_list(head);
5678        rcu_read_unlock();
5679}
5680
5681/**
5682 *      netif_receive_skb - process receive buffer from network
5683 *      @skb: buffer to process
5684 *
5685 *      netif_receive_skb() is the main receive data processing function.
5686 *      It always succeeds. The buffer may be dropped during processing
5687 *      for congestion control or by the protocol layers.
5688 *
5689 *      This function may only be called from softirq context and interrupts
5690 *      should be enabled.
5691 *
5692 *      Return values (usually ignored):
5693 *      NET_RX_SUCCESS: no congestion
5694 *      NET_RX_DROP: packet was dropped
5695 */
5696int netif_receive_skb(struct sk_buff *skb)
5697{
5698        int ret;
5699
5700        trace_netif_receive_skb_entry(skb);
5701
5702        ret = netif_receive_skb_internal(skb);
5703        trace_netif_receive_skb_exit(ret);
5704
5705        return ret;
5706}
5707EXPORT_SYMBOL(netif_receive_skb);
5708
5709/**
5710 *      netif_receive_skb_list - process many receive buffers from network
5711 *      @head: list of skbs to process.
5712 *
5713 *      Since return value of netif_receive_skb() is normally ignored, and
5714 *      wouldn't be meaningful for a list, this function returns void.
5715 *
5716 *      This function may only be called from softirq context and interrupts
5717 *      should be enabled.
5718 */
5719void netif_receive_skb_list(struct list_head *head)
5720{
5721        struct sk_buff *skb;
5722
5723        if (list_empty(head))
5724                return;
5725        if (trace_netif_receive_skb_list_entry_enabled()) {
5726                list_for_each_entry(skb, head, list)
5727                        trace_netif_receive_skb_list_entry(skb);
5728        }
5729        netif_receive_skb_list_internal(head);
5730        trace_netif_receive_skb_list_exit(0);
5731}
5732EXPORT_SYMBOL(netif_receive_skb_list);
5733
5734static DEFINE_PER_CPU(struct work_struct, flush_works);
5735
5736/* Network device is going away, flush any packets still pending */
5737static void flush_backlog(struct work_struct *work)
5738{
5739        struct sk_buff *skb, *tmp;
5740        struct softnet_data *sd;
5741
5742        local_bh_disable();
5743        sd = this_cpu_ptr(&softnet_data);
5744
5745        local_irq_disable();
5746        rps_lock(sd);
5747        skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5748                if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5749                        __skb_unlink(skb, &sd->input_pkt_queue);
5750                        dev_kfree_skb_irq(skb);
5751                        input_queue_head_incr(sd);
5752                }
5753        }
5754        rps_unlock(sd);
5755        local_irq_enable();
5756
5757        skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5758                if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5759                        __skb_unlink(skb, &sd->process_queue);
5760                        kfree_skb(skb);
5761                        input_queue_head_incr(sd);
5762                }
5763        }
5764        local_bh_enable();
5765}
5766
5767static bool flush_required(int cpu)
5768{
5769#if IS_ENABLED(CONFIG_RPS)
5770        struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5771        bool do_flush;
5772
5773        local_irq_disable();
5774        rps_lock(sd);
5775
5776        /* as insertion into process_queue happens with the rps lock held,
5777         * process_queue access may race only with dequeue
5778         */
5779        do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5780                   !skb_queue_empty_lockless(&sd->process_queue);
5781        rps_unlock(sd);
5782        local_irq_enable();
5783
5784        return do_flush;
5785#endif
5786        /* without RPS we can't safely check input_pkt_queue: during a
5787         * concurrent remote skb_queue_splice() we can detect as empty both
5788         * input_pkt_queue and process_queue even if the latter could end-up
5789         * containing a lot of packets.
5790         */
5791        return true;
5792}
5793
5794static void flush_all_backlogs(void)
5795{
5796        static cpumask_t flush_cpus;
5797        unsigned int cpu;
5798
5799        /* since we are under rtnl lock protection we can use static data
5800         * for the cpumask and avoid allocating on stack the possibly
5801         * large mask
5802         */
5803        ASSERT_RTNL();
5804
5805        cpus_read_lock();
5806
5807        cpumask_clear(&flush_cpus);
5808        for_each_online_cpu(cpu) {
5809                if (flush_required(cpu)) {
5810                        queue_work_on(cpu, system_highpri_wq,
5811                                      per_cpu_ptr(&flush_works, cpu));
5812                        cpumask_set_cpu(cpu, &flush_cpus);
5813                }
5814        }
5815
5816        /* we can have in flight packet[s] on the cpus we are not flushing,
5817         * synchronize_net() in unregister_netdevice_many() will take care of
5818         * them
5819         */
5820        for_each_cpu(cpu, &flush_cpus)
5821                flush_work(per_cpu_ptr(&flush_works, cpu));
5822
5823        cpus_read_unlock();
5824}
5825
5826/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
5827static void gro_normal_list(struct napi_struct *napi)
5828{
5829        if (!napi->rx_count)
5830                return;
5831        netif_receive_skb_list_internal(&napi->rx_list);
5832        INIT_LIST_HEAD(&napi->rx_list);
5833        napi->rx_count = 0;
5834}
5835
5836/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
5837 * pass the whole batch up to the stack.
5838 */
5839static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
5840{
5841        list_add_tail(&skb->list, &napi->rx_list);
5842        napi->rx_count += segs;
5843        if (napi->rx_count >= gro_normal_batch)
5844                gro_normal_list(napi);
5845}
5846
5847static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
5848{
5849        struct packet_offload *ptype;
5850        __be16 type = skb->protocol;
5851        struct list_head *head = &offload_base;
5852        int err = -ENOENT;
5853
5854        BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5855
5856        if (NAPI_GRO_CB(skb)->count == 1) {
5857                skb_shinfo(skb)->gso_size = 0;
5858                goto out;
5859        }
5860
5861        rcu_read_lock();
5862        list_for_each_entry_rcu(ptype, head, list) {
5863                if (ptype->type != type || !ptype->callbacks.gro_complete)
5864                        continue;
5865
5866                err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5867                                         ipv6_gro_complete, inet_gro_complete,
5868                                         skb, 0);
5869                break;
5870        }
5871        rcu_read_unlock();
5872
5873        if (err) {
5874                WARN_ON(&ptype->list == head);
5875                kfree_skb(skb);
5876                return NET_RX_SUCCESS;
5877        }
5878
5879out:
5880        gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
5881        return NET_RX_SUCCESS;
5882}
5883
5884static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
5885                                   bool flush_old)
5886{
5887        struct list_head *head = &napi->gro_hash[index].list;
5888        struct sk_buff *skb, *p;
5889
5890        list_for_each_entry_safe_reverse(skb, p, head, list) {
5891                if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5892                        return;
5893                skb_list_del_init(skb);
5894                napi_gro_complete(napi, skb);
5895                napi->gro_hash[index].count--;
5896        }
5897
5898        if (!napi->gro_hash[index].count)
5899                __clear_bit(index, &napi->gro_bitmask);
5900}
5901
5902/* napi->gro_hash[].list contains packets ordered by age.
5903 * youngest packets at the head of it.
5904 * Complete skbs in reverse order to reduce latencies.
5905 */
5906void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5907{
5908        unsigned long bitmask = napi->gro_bitmask;
5909        unsigned int i, base = ~0U;
5910
5911        while ((i = ffs(bitmask)) != 0) {
5912                bitmask >>= i;
5913                base += i;
5914                __napi_gro_flush_chain(napi, base, flush_old);
5915        }
5916}
5917EXPORT_SYMBOL(napi_gro_flush);
5918
5919static void gro_list_prepare(const struct list_head *head,
5920                             const struct sk_buff *skb)
5921{
5922        unsigned int maclen = skb->dev->hard_header_len;
5923        u32 hash = skb_get_hash_raw(skb);
5924        struct sk_buff *p;
5925
5926        list_for_each_entry(p, head, list) {
5927                unsigned long diffs;
5928
5929                NAPI_GRO_CB(p)->flush = 0;
5930
5931                if (hash != skb_get_hash_raw(p)) {
5932                        NAPI_GRO_CB(p)->same_flow = 0;
5933                        continue;
5934                }
5935
5936                diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
5937                diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
5938                if (skb_vlan_tag_present(p))
5939                        diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
5940                diffs |= skb_metadata_differs(p, skb);
5941                if (maclen == ETH_HLEN)
5942                        diffs |= compare_ether_header(skb_mac_header(p),
5943                                                      skb_mac_header(skb));
5944                else if (!diffs)
5945                        diffs = memcmp(skb_mac_header(p),
5946                                       skb_mac_header(skb),
5947                                       maclen);
5948
5949                /* in most common scenarions 'slow_gro' is 0
5950                 * otherwise we are already on some slower paths
5951                 * either skip all the infrequent tests altogether or
5952                 * avoid trying too hard to skip each of them individually
5953                 */
5954                if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
5955#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
5956                        struct tc_skb_ext *skb_ext;
5957                        struct tc_skb_ext *p_ext;
5958#endif
5959
5960                        diffs |= p->sk != skb->sk;
5961                        diffs |= skb_metadata_dst_cmp(p, skb);
5962                        diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
5963
5964#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
5965                        skb_ext = skb_ext_find(skb, TC_SKB_EXT);
5966                        p_ext = skb_ext_find(p, TC_SKB_EXT);
5967
5968                        diffs |= (!!p_ext) ^ (!!skb_ext);
5969                        if (!diffs && unlikely(skb_ext))
5970                                diffs |= p_ext->chain ^ skb_ext->chain;
5971#endif
5972                }
5973
5974                NAPI_GRO_CB(p)->same_flow = !diffs;
5975        }
5976}
5977
5978static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
5979{
5980        const struct skb_shared_info *pinfo = skb_shinfo(skb);
5981        const skb_frag_t *frag0 = &pinfo->frags[0];
5982
5983        NAPI_GRO_CB(skb)->data_offset = 0;
5984        NAPI_GRO_CB(skb)->frag0 = NULL;
5985        NAPI_GRO_CB(skb)->frag0_len = 0;
5986
5987        if (!skb_headlen(skb) && pinfo->nr_frags &&
5988            !PageHighMem(skb_frag_page(frag0)) &&
5989            (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
5990                NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
5991                NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5992                                                    skb_frag_size(frag0),
5993                                                    skb->end - skb->tail);
5994        }
5995}
5996
5997static void gro_pull_from_frag0(struct sk_buff *skb, int grow)