1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75#include <linux/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/sched/mm.h>
85#include <linux/mutex.h>
86#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
95#include <linux/ethtool.h>
96#include <linux/notifier.h>
97#include <linux/skbuff.h>
98#include <linux/bpf.h>
99#include <linux/bpf_trace.h>
100#include <net/net_namespace.h>
101#include <net/sock.h>
102#include <net/busy_poll.h>
103#include <linux/rtnetlink.h>
104#include <linux/stat.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/pkt_sched.h>
108#include <net/pkt_cls.h>
109#include <net/checksum.h>
110#include <net/xfrm.h>
111#include <linux/highmem.h>
112#include <linux/init.h>
113#include <linux/module.h>
114#include <linux/netpoll.h>
115#include <linux/rcupdate.h>
116#include <linux/delay.h>
117#include <net/iw_handler.h>
118#include <asm/current.h>
119#include <linux/audit.h>
120#include <linux/dmaengine.h>
121#include <linux/err.h>
122#include <linux/ctype.h>
123#include <linux/if_arp.h>
124#include <linux/if_vlan.h>
125#include <linux/ip.h>
126#include <net/ip.h>
127#include <net/mpls.h>
128#include <linux/ipv6.h>
129#include <linux/in.h>
130#include <linux/jhash.h>
131#include <linux/random.h>
132#include <trace/events/napi.h>
133#include <trace/events/net.h>
134#include <trace/events/skb.h>
135#include <linux/pci.h>
136#include <linux/inetdevice.h>
137#include <linux/cpu_rmap.h>
138#include <linux/static_key.h>
139#include <linux/hashtable.h>
140#include <linux/vmalloc.h>
141#include <linux/if_macvlan.h>
142#include <linux/errqueue.h>
143#include <linux/hrtimer.h>
144#include <linux/netfilter_ingress.h>
145#include <linux/crash_dump.h>
146#include <linux/sctp.h>
147#include <net/udp_tunnel.h>
148#include <linux/net_namespace.h>
149
150#include "net-sysfs.h"
151
152
153#define MAX_GRO_SKBS 8
154
155
156#define GRO_MAX_HEAD (MAX_HEADER + 128)
157
158static DEFINE_SPINLOCK(ptype_lock);
159static DEFINE_SPINLOCK(offload_lock);
160struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
161struct list_head ptype_all __read_mostly;
162static struct list_head offload_base __read_mostly;
163
164static int netif_rx_internal(struct sk_buff *skb);
165static int call_netdevice_notifiers_info(unsigned long val,
166 struct netdev_notifier_info *info);
167static struct napi_struct *napi_by_id(unsigned int napi_id);
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188DEFINE_RWLOCK(dev_base_lock);
189EXPORT_SYMBOL(dev_base_lock);
190
191static DEFINE_MUTEX(ifalias_mutex);
192
193
194static DEFINE_SPINLOCK(napi_hash_lock);
195
196static unsigned int napi_gen_id = NR_CPUS;
197static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
198
199static seqcount_t devnet_rename_seq;
200
201static inline void dev_base_seq_inc(struct net *net)
202{
203 while (++net->dev_base_seq == 0)
204 ;
205}
206
207static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
208{
209 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
210
211 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
212}
213
214static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
215{
216 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
217}
218
219static inline void rps_lock(struct softnet_data *sd)
220{
221#ifdef CONFIG_RPS
222 spin_lock(&sd->input_pkt_queue.lock);
223#endif
224}
225
226static inline void rps_unlock(struct softnet_data *sd)
227{
228#ifdef CONFIG_RPS
229 spin_unlock(&sd->input_pkt_queue.lock);
230#endif
231}
232
233
234static void list_netdevice(struct net_device *dev)
235{
236 struct net *net = dev_net(dev);
237
238 ASSERT_RTNL();
239
240 write_lock_bh(&dev_base_lock);
241 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
242 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
243 hlist_add_head_rcu(&dev->index_hlist,
244 dev_index_hash(net, dev->ifindex));
245 write_unlock_bh(&dev_base_lock);
246
247 dev_base_seq_inc(net);
248}
249
250
251
252
253static void unlist_netdevice(struct net_device *dev)
254{
255 ASSERT_RTNL();
256
257
258 write_lock_bh(&dev_base_lock);
259 list_del_rcu(&dev->dev_list);
260 hlist_del_rcu(&dev->name_hlist);
261 hlist_del_rcu(&dev->index_hlist);
262 write_unlock_bh(&dev_base_lock);
263
264 dev_base_seq_inc(dev_net(dev));
265}
266
267
268
269
270
271static RAW_NOTIFIER_HEAD(netdev_chain);
272
273
274
275
276
277
278DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
279EXPORT_PER_CPU_SYMBOL(softnet_data);
280
281#ifdef CONFIG_LOCKDEP
282
283
284
285
286static const unsigned short netdev_lock_type[] = {
287 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
288 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
289 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
290 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
291 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
292 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
293 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
294 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
295 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
296 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
297 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
298 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
299 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
300 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
301 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
302
303static const char *const netdev_lock_name[] = {
304 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
305 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
306 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
307 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
308 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
309 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
310 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
311 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
312 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
313 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
314 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
315 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
316 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
317 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
318 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
319
320static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
321static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
322
323static inline unsigned short netdev_lock_pos(unsigned short dev_type)
324{
325 int i;
326
327 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
328 if (netdev_lock_type[i] == dev_type)
329 return i;
330
331 return ARRAY_SIZE(netdev_lock_type) - 1;
332}
333
334static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
335 unsigned short dev_type)
336{
337 int i;
338
339 i = netdev_lock_pos(dev_type);
340 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
341 netdev_lock_name[i]);
342}
343
344static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
345{
346 int i;
347
348 i = netdev_lock_pos(dev->type);
349 lockdep_set_class_and_name(&dev->addr_list_lock,
350 &netdev_addr_lock_key[i],
351 netdev_lock_name[i]);
352}
353#else
354static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
355 unsigned short dev_type)
356{
357}
358static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
359{
360}
361#endif
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386static inline struct list_head *ptype_head(const struct packet_type *pt)
387{
388 if (pt->type == htons(ETH_P_ALL))
389 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
390 else
391 return pt->dev ? &pt->dev->ptype_specific :
392 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
393}
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408void dev_add_pack(struct packet_type *pt)
409{
410 struct list_head *head = ptype_head(pt);
411
412 spin_lock(&ptype_lock);
413 list_add_rcu(&pt->list, head);
414 spin_unlock(&ptype_lock);
415}
416EXPORT_SYMBOL(dev_add_pack);
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431void __dev_remove_pack(struct packet_type *pt)
432{
433 struct list_head *head = ptype_head(pt);
434 struct packet_type *pt1;
435
436 spin_lock(&ptype_lock);
437
438 list_for_each_entry(pt1, head, list) {
439 if (pt == pt1) {
440 list_del_rcu(&pt->list);
441 goto out;
442 }
443 }
444
445 pr_warn("dev_remove_pack: %p not found\n", pt);
446out:
447 spin_unlock(&ptype_lock);
448}
449EXPORT_SYMBOL(__dev_remove_pack);
450
451
452
453
454
455
456
457
458
459
460
461
462
463void dev_remove_pack(struct packet_type *pt)
464{
465 __dev_remove_pack(pt);
466
467 synchronize_net();
468}
469EXPORT_SYMBOL(dev_remove_pack);
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484void dev_add_offload(struct packet_offload *po)
485{
486 struct packet_offload *elem;
487
488 spin_lock(&offload_lock);
489 list_for_each_entry(elem, &offload_base, list) {
490 if (po->priority < elem->priority)
491 break;
492 }
493 list_add_rcu(&po->list, elem->list.prev);
494 spin_unlock(&offload_lock);
495}
496EXPORT_SYMBOL(dev_add_offload);
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511static void __dev_remove_offload(struct packet_offload *po)
512{
513 struct list_head *head = &offload_base;
514 struct packet_offload *po1;
515
516 spin_lock(&offload_lock);
517
518 list_for_each_entry(po1, head, list) {
519 if (po == po1) {
520 list_del_rcu(&po->list);
521 goto out;
522 }
523 }
524
525 pr_warn("dev_remove_offload: %p not found\n", po);
526out:
527 spin_unlock(&offload_lock);
528}
529
530
531
532
533
534
535
536
537
538
539
540
541
542void dev_remove_offload(struct packet_offload *po)
543{
544 __dev_remove_offload(po);
545
546 synchronize_net();
547}
548EXPORT_SYMBOL(dev_remove_offload);
549
550
551
552
553
554
555
556
557static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
558
559
560
561
562
563
564
565
566
567
568static int netdev_boot_setup_add(char *name, struct ifmap *map)
569{
570 struct netdev_boot_setup *s;
571 int i;
572
573 s = dev_boot_setup;
574 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
575 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
576 memset(s[i].name, 0, sizeof(s[i].name));
577 strlcpy(s[i].name, name, IFNAMSIZ);
578 memcpy(&s[i].map, map, sizeof(s[i].map));
579 break;
580 }
581 }
582
583 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
584}
585
586
587
588
589
590
591
592
593
594
595int netdev_boot_setup_check(struct net_device *dev)
596{
597 struct netdev_boot_setup *s = dev_boot_setup;
598 int i;
599
600 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
601 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
602 !strcmp(dev->name, s[i].name)) {
603 dev->irq = s[i].map.irq;
604 dev->base_addr = s[i].map.base_addr;
605 dev->mem_start = s[i].map.mem_start;
606 dev->mem_end = s[i].map.mem_end;
607 return 1;
608 }
609 }
610 return 0;
611}
612EXPORT_SYMBOL(netdev_boot_setup_check);
613
614
615
616
617
618
619
620
621
622
623
624
625unsigned long netdev_boot_base(const char *prefix, int unit)
626{
627 const struct netdev_boot_setup *s = dev_boot_setup;
628 char name[IFNAMSIZ];
629 int i;
630
631 sprintf(name, "%s%d", prefix, unit);
632
633
634
635
636
637 if (__dev_get_by_name(&init_net, name))
638 return 1;
639
640 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
641 if (!strcmp(name, s[i].name))
642 return s[i].map.base_addr;
643 return 0;
644}
645
646
647
648
649int __init netdev_boot_setup(char *str)
650{
651 int ints[5];
652 struct ifmap map;
653
654 str = get_options(str, ARRAY_SIZE(ints), ints);
655 if (!str || !*str)
656 return 0;
657
658
659 memset(&map, 0, sizeof(map));
660 if (ints[0] > 0)
661 map.irq = ints[1];
662 if (ints[0] > 1)
663 map.base_addr = ints[2];
664 if (ints[0] > 2)
665 map.mem_start = ints[3];
666 if (ints[0] > 3)
667 map.mem_end = ints[4];
668
669
670 return netdev_boot_setup_add(str, &map);
671}
672
673__setup("netdev=", netdev_boot_setup);
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689int dev_get_iflink(const struct net_device *dev)
690{
691 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
692 return dev->netdev_ops->ndo_get_iflink(dev);
693
694 return dev->ifindex;
695}
696EXPORT_SYMBOL(dev_get_iflink);
697
698
699
700
701
702
703
704
705
706
707int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
708{
709 struct ip_tunnel_info *info;
710
711 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
712 return -EINVAL;
713
714 info = skb_tunnel_info_unclone(skb);
715 if (!info)
716 return -ENOMEM;
717 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
718 return -EINVAL;
719
720 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
721}
722EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
723
724
725
726
727
728
729
730
731
732
733
734
735
736struct net_device *__dev_get_by_name(struct net *net, const char *name)
737{
738 struct net_device *dev;
739 struct hlist_head *head = dev_name_hash(net, name);
740
741 hlist_for_each_entry(dev, head, name_hlist)
742 if (!strncmp(dev->name, name, IFNAMSIZ))
743 return dev;
744
745 return NULL;
746}
747EXPORT_SYMBOL(__dev_get_by_name);
748
749
750
751
752
753
754
755
756
757
758
759
760
761struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
762{
763 struct net_device *dev;
764 struct hlist_head *head = dev_name_hash(net, name);
765
766 hlist_for_each_entry_rcu(dev, head, name_hlist)
767 if (!strncmp(dev->name, name, IFNAMSIZ))
768 return dev;
769
770 return NULL;
771}
772EXPORT_SYMBOL(dev_get_by_name_rcu);
773
774
775
776
777
778
779
780
781
782
783
784
785
786struct net_device *dev_get_by_name(struct net *net, const char *name)
787{
788 struct net_device *dev;
789
790 rcu_read_lock();
791 dev = dev_get_by_name_rcu(net, name);
792 if (dev)
793 dev_hold(dev);
794 rcu_read_unlock();
795 return dev;
796}
797EXPORT_SYMBOL(dev_get_by_name);
798
799
800
801
802
803
804
805
806
807
808
809
810
811struct net_device *__dev_get_by_index(struct net *net, int ifindex)
812{
813 struct net_device *dev;
814 struct hlist_head *head = dev_index_hash(net, ifindex);
815
816 hlist_for_each_entry(dev, head, index_hlist)
817 if (dev->ifindex == ifindex)
818 return dev;
819
820 return NULL;
821}
822EXPORT_SYMBOL(__dev_get_by_index);
823
824
825
826
827
828
829
830
831
832
833
834
835struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
836{
837 struct net_device *dev;
838 struct hlist_head *head = dev_index_hash(net, ifindex);
839
840 hlist_for_each_entry_rcu(dev, head, index_hlist)
841 if (dev->ifindex == ifindex)
842 return dev;
843
844 return NULL;
845}
846EXPORT_SYMBOL(dev_get_by_index_rcu);
847
848
849
850
851
852
853
854
855
856
857
858
859
860struct net_device *dev_get_by_index(struct net *net, int ifindex)
861{
862 struct net_device *dev;
863
864 rcu_read_lock();
865 dev = dev_get_by_index_rcu(net, ifindex);
866 if (dev)
867 dev_hold(dev);
868 rcu_read_unlock();
869 return dev;
870}
871EXPORT_SYMBOL(dev_get_by_index);
872
873
874
875
876
877
878
879
880
881
882
883struct net_device *dev_get_by_napi_id(unsigned int napi_id)
884{
885 struct napi_struct *napi;
886
887 WARN_ON_ONCE(!rcu_read_lock_held());
888
889 if (napi_id < MIN_NAPI_ID)
890 return NULL;
891
892 napi = napi_by_id(napi_id);
893
894 return napi ? napi->dev : NULL;
895}
896EXPORT_SYMBOL(dev_get_by_napi_id);
897
898
899
900
901
902
903
904
905
906
907
908int netdev_get_name(struct net *net, char *name, int ifindex)
909{
910 struct net_device *dev;
911 unsigned int seq;
912
913retry:
914 seq = raw_seqcount_begin(&devnet_rename_seq);
915 rcu_read_lock();
916 dev = dev_get_by_index_rcu(net, ifindex);
917 if (!dev) {
918 rcu_read_unlock();
919 return -ENODEV;
920 }
921
922 strcpy(name, dev->name);
923 rcu_read_unlock();
924 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
925 cond_resched();
926 goto retry;
927 }
928
929 return 0;
930}
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
947 const char *ha)
948{
949 struct net_device *dev;
950
951 for_each_netdev_rcu(net, dev)
952 if (dev->type == type &&
953 !memcmp(dev->dev_addr, ha, dev->addr_len))
954 return dev;
955
956 return NULL;
957}
958EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
959
960struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
961{
962 struct net_device *dev;
963
964 ASSERT_RTNL();
965 for_each_netdev(net, dev)
966 if (dev->type == type)
967 return dev;
968
969 return NULL;
970}
971EXPORT_SYMBOL(__dev_getfirstbyhwtype);
972
973struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
974{
975 struct net_device *dev, *ret = NULL;
976
977 rcu_read_lock();
978 for_each_netdev_rcu(net, dev)
979 if (dev->type == type) {
980 dev_hold(dev);
981 ret = dev;
982 break;
983 }
984 rcu_read_unlock();
985 return ret;
986}
987EXPORT_SYMBOL(dev_getfirstbyhwtype);
988
989
990
991
992
993
994
995
996
997
998
999
1000struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1001 unsigned short mask)
1002{
1003 struct net_device *dev, *ret;
1004
1005 ASSERT_RTNL();
1006
1007 ret = NULL;
1008 for_each_netdev(net, dev) {
1009 if (((dev->flags ^ if_flags) & mask) == 0) {
1010 ret = dev;
1011 break;
1012 }
1013 }
1014 return ret;
1015}
1016EXPORT_SYMBOL(__dev_get_by_flags);
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026bool dev_valid_name(const char *name)
1027{
1028 if (*name == '\0')
1029 return false;
1030 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1031 return false;
1032 if (!strcmp(name, ".") || !strcmp(name, ".."))
1033 return false;
1034
1035 while (*name) {
1036 if (*name == '/' || *name == ':' || isspace(*name))
1037 return false;
1038 name++;
1039 }
1040 return true;
1041}
1042EXPORT_SYMBOL(dev_valid_name);
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1060{
1061 int i = 0;
1062 const char *p;
1063 const int max_netdevices = 8*PAGE_SIZE;
1064 unsigned long *inuse;
1065 struct net_device *d;
1066
1067 if (!dev_valid_name(name))
1068 return -EINVAL;
1069
1070 p = strchr(name, '%');
1071 if (p) {
1072
1073
1074
1075
1076
1077 if (p[1] != 'd' || strchr(p + 2, '%'))
1078 return -EINVAL;
1079
1080
1081 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1082 if (!inuse)
1083 return -ENOMEM;
1084
1085 for_each_netdev(net, d) {
1086 if (!sscanf(d->name, name, &i))
1087 continue;
1088 if (i < 0 || i >= max_netdevices)
1089 continue;
1090
1091
1092 snprintf(buf, IFNAMSIZ, name, i);
1093 if (!strncmp(buf, d->name, IFNAMSIZ))
1094 set_bit(i, inuse);
1095 }
1096
1097 i = find_first_zero_bit(inuse, max_netdevices);
1098 free_page((unsigned long) inuse);
1099 }
1100
1101 snprintf(buf, IFNAMSIZ, name, i);
1102 if (!__dev_get_by_name(net, buf))
1103 return i;
1104
1105
1106
1107
1108
1109 return -ENFILE;
1110}
1111
1112static int dev_alloc_name_ns(struct net *net,
1113 struct net_device *dev,
1114 const char *name)
1115{
1116 char buf[IFNAMSIZ];
1117 int ret;
1118
1119 BUG_ON(!net);
1120 ret = __dev_alloc_name(net, name, buf);
1121 if (ret >= 0)
1122 strlcpy(dev->name, buf, IFNAMSIZ);
1123 return ret;
1124}
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140int dev_alloc_name(struct net_device *dev, const char *name)
1141{
1142 return dev_alloc_name_ns(dev_net(dev), dev, name);
1143}
1144EXPORT_SYMBOL(dev_alloc_name);
1145
1146int dev_get_valid_name(struct net *net, struct net_device *dev,
1147 const char *name)
1148{
1149 BUG_ON(!net);
1150
1151 if (!dev_valid_name(name))
1152 return -EINVAL;
1153
1154 if (strchr(name, '%'))
1155 return dev_alloc_name_ns(net, dev, name);
1156 else if (__dev_get_by_name(net, name))
1157 return -EEXIST;
1158 else if (dev->name != name)
1159 strlcpy(dev->name, name, IFNAMSIZ);
1160
1161 return 0;
1162}
1163EXPORT_SYMBOL(dev_get_valid_name);
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173int dev_change_name(struct net_device *dev, const char *newname)
1174{
1175 unsigned char old_assign_type;
1176 char oldname[IFNAMSIZ];
1177 int err = 0;
1178 int ret;
1179 struct net *net;
1180
1181 ASSERT_RTNL();
1182 BUG_ON(!dev_net(dev));
1183
1184 net = dev_net(dev);
1185 if (dev->flags & IFF_UP)
1186 return -EBUSY;
1187
1188 write_seqcount_begin(&devnet_rename_seq);
1189
1190 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1191 write_seqcount_end(&devnet_rename_seq);
1192 return 0;
1193 }
1194
1195 memcpy(oldname, dev->name, IFNAMSIZ);
1196
1197 err = dev_get_valid_name(net, dev, newname);
1198 if (err < 0) {
1199 write_seqcount_end(&devnet_rename_seq);
1200 return err;
1201 }
1202
1203 if (oldname[0] && !strchr(oldname, '%'))
1204 netdev_info(dev, "renamed from %s\n", oldname);
1205
1206 old_assign_type = dev->name_assign_type;
1207 dev->name_assign_type = NET_NAME_RENAMED;
1208
1209rollback:
1210 ret = device_rename(&dev->dev, dev->name);
1211 if (ret) {
1212 memcpy(dev->name, oldname, IFNAMSIZ);
1213 dev->name_assign_type = old_assign_type;
1214 write_seqcount_end(&devnet_rename_seq);
1215 return ret;
1216 }
1217
1218 write_seqcount_end(&devnet_rename_seq);
1219
1220 netdev_adjacent_rename_links(dev, oldname);
1221
1222 write_lock_bh(&dev_base_lock);
1223 hlist_del_rcu(&dev->name_hlist);
1224 write_unlock_bh(&dev_base_lock);
1225
1226 synchronize_rcu();
1227
1228 write_lock_bh(&dev_base_lock);
1229 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1230 write_unlock_bh(&dev_base_lock);
1231
1232 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1233 ret = notifier_to_errno(ret);
1234
1235 if (ret) {
1236
1237 if (err >= 0) {
1238 err = ret;
1239 write_seqcount_begin(&devnet_rename_seq);
1240 memcpy(dev->name, oldname, IFNAMSIZ);
1241 memcpy(oldname, newname, IFNAMSIZ);
1242 dev->name_assign_type = old_assign_type;
1243 old_assign_type = NET_NAME_RENAMED;
1244 goto rollback;
1245 } else {
1246 pr_err("%s: name change rollback failed: %d\n",
1247 dev->name, ret);
1248 }
1249 }
1250
1251 return err;
1252}
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1263{
1264 struct dev_ifalias *new_alias = NULL;
1265
1266 if (len >= IFALIASZ)
1267 return -EINVAL;
1268
1269 if (len) {
1270 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1271 if (!new_alias)
1272 return -ENOMEM;
1273
1274 memcpy(new_alias->ifalias, alias, len);
1275 new_alias->ifalias[len] = 0;
1276 }
1277
1278 mutex_lock(&ifalias_mutex);
1279 rcu_swap_protected(dev->ifalias, new_alias,
1280 mutex_is_locked(&ifalias_mutex));
1281 mutex_unlock(&ifalias_mutex);
1282
1283 if (new_alias)
1284 kfree_rcu(new_alias, rcuhead);
1285
1286 return len;
1287}
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1299{
1300 const struct dev_ifalias *alias;
1301 int ret = 0;
1302
1303 rcu_read_lock();
1304 alias = rcu_dereference(dev->ifalias);
1305 if (alias)
1306 ret = snprintf(name, len, "%s", alias->ifalias);
1307 rcu_read_unlock();
1308
1309 return ret;
1310}
1311
1312
1313
1314
1315
1316
1317
1318void netdev_features_change(struct net_device *dev)
1319{
1320 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1321}
1322EXPORT_SYMBOL(netdev_features_change);
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332void netdev_state_change(struct net_device *dev)
1333{
1334 if (dev->flags & IFF_UP) {
1335 struct netdev_notifier_change_info change_info = {
1336 .info.dev = dev,
1337 };
1338
1339 call_netdevice_notifiers_info(NETDEV_CHANGE,
1340 &change_info.info);
1341 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1342 }
1343}
1344EXPORT_SYMBOL(netdev_state_change);
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356void netdev_notify_peers(struct net_device *dev)
1357{
1358 rtnl_lock();
1359 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1360 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1361 rtnl_unlock();
1362}
1363EXPORT_SYMBOL(netdev_notify_peers);
1364
1365static int __dev_open(struct net_device *dev)
1366{
1367 const struct net_device_ops *ops = dev->netdev_ops;
1368 int ret;
1369
1370 ASSERT_RTNL();
1371
1372 if (!netif_device_present(dev))
1373 return -ENODEV;
1374
1375
1376
1377
1378
1379 netpoll_poll_disable(dev);
1380
1381 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1382 ret = notifier_to_errno(ret);
1383 if (ret)
1384 return ret;
1385
1386 set_bit(__LINK_STATE_START, &dev->state);
1387
1388 if (ops->ndo_validate_addr)
1389 ret = ops->ndo_validate_addr(dev);
1390
1391 if (!ret && ops->ndo_open)
1392 ret = ops->ndo_open(dev);
1393
1394 netpoll_poll_enable(dev);
1395
1396 if (ret)
1397 clear_bit(__LINK_STATE_START, &dev->state);
1398 else {
1399 dev->flags |= IFF_UP;
1400 dev_set_rx_mode(dev);
1401 dev_activate(dev);
1402 add_device_randomness(dev->dev_addr, dev->addr_len);
1403 }
1404
1405 return ret;
1406}
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420int dev_open(struct net_device *dev)
1421{
1422 int ret;
1423
1424 if (dev->flags & IFF_UP)
1425 return 0;
1426
1427 ret = __dev_open(dev);
1428 if (ret < 0)
1429 return ret;
1430
1431 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1432 call_netdevice_notifiers(NETDEV_UP, dev);
1433
1434 return ret;
1435}
1436EXPORT_SYMBOL(dev_open);
1437
1438static void __dev_close_many(struct list_head *head)
1439{
1440 struct net_device *dev;
1441
1442 ASSERT_RTNL();
1443 might_sleep();
1444
1445 list_for_each_entry(dev, head, close_list) {
1446
1447 netpoll_poll_disable(dev);
1448
1449 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1450
1451 clear_bit(__LINK_STATE_START, &dev->state);
1452
1453
1454
1455
1456
1457
1458
1459 smp_mb__after_atomic();
1460 }
1461
1462 dev_deactivate_many(head);
1463
1464 list_for_each_entry(dev, head, close_list) {
1465 const struct net_device_ops *ops = dev->netdev_ops;
1466
1467
1468
1469
1470
1471
1472
1473
1474 if (ops->ndo_stop)
1475 ops->ndo_stop(dev);
1476
1477 dev->flags &= ~IFF_UP;
1478 netpoll_poll_enable(dev);
1479 }
1480}
1481
1482static void __dev_close(struct net_device *dev)
1483{
1484 LIST_HEAD(single);
1485
1486 list_add(&dev->close_list, &single);
1487 __dev_close_many(&single);
1488 list_del(&single);
1489}
1490
1491void dev_close_many(struct list_head *head, bool unlink)
1492{
1493 struct net_device *dev, *tmp;
1494
1495
1496 list_for_each_entry_safe(dev, tmp, head, close_list)
1497 if (!(dev->flags & IFF_UP))
1498 list_del_init(&dev->close_list);
1499
1500 __dev_close_many(head);
1501
1502 list_for_each_entry_safe(dev, tmp, head, close_list) {
1503 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1504 call_netdevice_notifiers(NETDEV_DOWN, dev);
1505 if (unlink)
1506 list_del_init(&dev->close_list);
1507 }
1508}
1509EXPORT_SYMBOL(dev_close_many);
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520void dev_close(struct net_device *dev)
1521{
1522 if (dev->flags & IFF_UP) {
1523 LIST_HEAD(single);
1524
1525 list_add(&dev->close_list, &single);
1526 dev_close_many(&single, true);
1527 list_del(&single);
1528 }
1529}
1530EXPORT_SYMBOL(dev_close);
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541void dev_disable_lro(struct net_device *dev)
1542{
1543 struct net_device *lower_dev;
1544 struct list_head *iter;
1545
1546 dev->wanted_features &= ~NETIF_F_LRO;
1547 netdev_update_features(dev);
1548
1549 if (unlikely(dev->features & NETIF_F_LRO))
1550 netdev_WARN(dev, "failed to disable LRO!\n");
1551
1552 netdev_for_each_lower_dev(dev, lower_dev, iter)
1553 dev_disable_lro(lower_dev);
1554}
1555EXPORT_SYMBOL(dev_disable_lro);
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565static void dev_disable_gro_hw(struct net_device *dev)
1566{
1567 dev->wanted_features &= ~NETIF_F_GRO_HW;
1568 netdev_update_features(dev);
1569
1570 if (unlikely(dev->features & NETIF_F_GRO_HW))
1571 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1572}
1573
1574const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1575{
1576#define N(val) \
1577 case NETDEV_##val: \
1578 return "NETDEV_" __stringify(val);
1579 switch (cmd) {
1580 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1581 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1582 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1583 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1584 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1585 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1586 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1587 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1588 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1589 };
1590#undef N
1591 return "UNKNOWN_NETDEV_EVENT";
1592}
1593EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1594
1595static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1596 struct net_device *dev)
1597{
1598 struct netdev_notifier_info info = {
1599 .dev = dev,
1600 };
1601
1602 return nb->notifier_call(nb, val, &info);
1603}
1604
1605static int dev_boot_phase = 1;
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621int register_netdevice_notifier(struct notifier_block *nb)
1622{
1623 struct net_device *dev;
1624 struct net_device *last;
1625 struct net *net;
1626 int err;
1627
1628
1629 down_write(&pernet_ops_rwsem);
1630 rtnl_lock();
1631 err = raw_notifier_chain_register(&netdev_chain, nb);
1632 if (err)
1633 goto unlock;
1634 if (dev_boot_phase)
1635 goto unlock;
1636 for_each_net(net) {
1637 for_each_netdev(net, dev) {
1638 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1639 err = notifier_to_errno(err);
1640 if (err)
1641 goto rollback;
1642
1643 if (!(dev->flags & IFF_UP))
1644 continue;
1645
1646 call_netdevice_notifier(nb, NETDEV_UP, dev);
1647 }
1648 }
1649
1650unlock:
1651 rtnl_unlock();
1652 up_write(&pernet_ops_rwsem);
1653 return err;
1654
1655rollback:
1656 last = dev;
1657 for_each_net(net) {
1658 for_each_netdev(net, dev) {
1659 if (dev == last)
1660 goto outroll;
1661
1662 if (dev->flags & IFF_UP) {
1663 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1664 dev);
1665 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1666 }
1667 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1668 }
1669 }
1670
1671outroll:
1672 raw_notifier_chain_unregister(&netdev_chain, nb);
1673 goto unlock;
1674}
1675EXPORT_SYMBOL(register_netdevice_notifier);
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691int unregister_netdevice_notifier(struct notifier_block *nb)
1692{
1693 struct net_device *dev;
1694 struct net *net;
1695 int err;
1696
1697
1698 down_write(&pernet_ops_rwsem);
1699 rtnl_lock();
1700 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1701 if (err)
1702 goto unlock;
1703
1704 for_each_net(net) {
1705 for_each_netdev(net, dev) {
1706 if (dev->flags & IFF_UP) {
1707 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1708 dev);
1709 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1710 }
1711 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1712 }
1713 }
1714unlock:
1715 rtnl_unlock();
1716 up_write(&pernet_ops_rwsem);
1717 return err;
1718}
1719EXPORT_SYMBOL(unregister_netdevice_notifier);
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730static int call_netdevice_notifiers_info(unsigned long val,
1731 struct netdev_notifier_info *info)
1732{
1733 ASSERT_RTNL();
1734 return raw_notifier_call_chain(&netdev_chain, val, info);
1735}
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1747{
1748 struct netdev_notifier_info info = {
1749 .dev = dev,
1750 };
1751
1752 return call_netdevice_notifiers_info(val, &info);
1753}
1754EXPORT_SYMBOL(call_netdevice_notifiers);
1755
1756#ifdef CONFIG_NET_INGRESS
1757static struct static_key ingress_needed __read_mostly;
1758
1759void net_inc_ingress_queue(void)
1760{
1761 static_key_slow_inc(&ingress_needed);
1762}
1763EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1764
1765void net_dec_ingress_queue(void)
1766{
1767 static_key_slow_dec(&ingress_needed);
1768}
1769EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1770#endif
1771
1772#ifdef CONFIG_NET_EGRESS
1773static struct static_key egress_needed __read_mostly;
1774
1775void net_inc_egress_queue(void)
1776{
1777 static_key_slow_inc(&egress_needed);
1778}
1779EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1780
1781void net_dec_egress_queue(void)
1782{
1783 static_key_slow_dec(&egress_needed);
1784}
1785EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1786#endif
1787
1788static struct static_key netstamp_needed __read_mostly;
1789#ifdef HAVE_JUMP_LABEL
1790static atomic_t netstamp_needed_deferred;
1791static atomic_t netstamp_wanted;
1792static void netstamp_clear(struct work_struct *work)
1793{
1794 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1795 int wanted;
1796
1797 wanted = atomic_add_return(deferred, &netstamp_wanted);
1798 if (wanted > 0)
1799 static_key_enable(&netstamp_needed);
1800 else
1801 static_key_disable(&netstamp_needed);
1802}
1803static DECLARE_WORK(netstamp_work, netstamp_clear);
1804#endif
1805
1806void net_enable_timestamp(void)
1807{
1808#ifdef HAVE_JUMP_LABEL
1809 int wanted;
1810
1811 while (1) {
1812 wanted = atomic_read(&netstamp_wanted);
1813 if (wanted <= 0)
1814 break;
1815 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1816 return;
1817 }
1818 atomic_inc(&netstamp_needed_deferred);
1819 schedule_work(&netstamp_work);
1820#else
1821 static_key_slow_inc(&netstamp_needed);
1822#endif
1823}
1824EXPORT_SYMBOL(net_enable_timestamp);
1825
1826void net_disable_timestamp(void)
1827{
1828#ifdef HAVE_JUMP_LABEL
1829 int wanted;
1830
1831 while (1) {
1832 wanted = atomic_read(&netstamp_wanted);
1833 if (wanted <= 1)
1834 break;
1835 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1836 return;
1837 }
1838 atomic_dec(&netstamp_needed_deferred);
1839 schedule_work(&netstamp_work);
1840#else
1841 static_key_slow_dec(&netstamp_needed);
1842#endif
1843}
1844EXPORT_SYMBOL(net_disable_timestamp);
1845
1846static inline void net_timestamp_set(struct sk_buff *skb)
1847{
1848 skb->tstamp = 0;
1849 if (static_key_false(&netstamp_needed))
1850 __net_timestamp(skb);
1851}
1852
1853#define net_timestamp_check(COND, SKB) \
1854 if (static_key_false(&netstamp_needed)) { \
1855 if ((COND) && !(SKB)->tstamp) \
1856 __net_timestamp(SKB); \
1857 } \
1858
1859bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1860{
1861 unsigned int len;
1862
1863 if (!(dev->flags & IFF_UP))
1864 return false;
1865
1866 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1867 if (skb->len <= len)
1868 return true;
1869
1870
1871
1872
1873 if (skb_is_gso(skb))
1874 return true;
1875
1876 return false;
1877}
1878EXPORT_SYMBOL_GPL(is_skb_forwardable);
1879
1880int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1881{
1882 int ret = ____dev_forward_skb(dev, skb);
1883
1884 if (likely(!ret)) {
1885 skb->protocol = eth_type_trans(skb, dev);
1886 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1887 }
1888
1889 return ret;
1890}
1891EXPORT_SYMBOL_GPL(__dev_forward_skb);
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1912{
1913 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1914}
1915EXPORT_SYMBOL_GPL(dev_forward_skb);
1916
1917static inline int deliver_skb(struct sk_buff *skb,
1918 struct packet_type *pt_prev,
1919 struct net_device *orig_dev)
1920{
1921 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1922 return -ENOMEM;
1923 refcount_inc(&skb->users);
1924 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1925}
1926
1927static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1928 struct packet_type **pt,
1929 struct net_device *orig_dev,
1930 __be16 type,
1931 struct list_head *ptype_list)
1932{
1933 struct packet_type *ptype, *pt_prev = *pt;
1934
1935 list_for_each_entry_rcu(ptype, ptype_list, list) {
1936 if (ptype->type != type)
1937 continue;
1938 if (pt_prev)
1939 deliver_skb(skb, pt_prev, orig_dev);
1940 pt_prev = ptype;
1941 }
1942 *pt = pt_prev;
1943}
1944
1945static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1946{
1947 if (!ptype->af_packet_priv || !skb->sk)
1948 return false;
1949
1950 if (ptype->id_match)
1951 return ptype->id_match(ptype, skb->sk);
1952 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1953 return true;
1954
1955 return false;
1956}
1957
1958
1959
1960
1961
1962
1963void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1964{
1965 struct packet_type *ptype;
1966 struct sk_buff *skb2 = NULL;
1967 struct packet_type *pt_prev = NULL;
1968 struct list_head *ptype_list = &ptype_all;
1969
1970 rcu_read_lock();
1971again:
1972 list_for_each_entry_rcu(ptype, ptype_list, list) {
1973
1974
1975
1976 if (skb_loop_sk(ptype, skb))
1977 continue;
1978
1979 if (pt_prev) {
1980 deliver_skb(skb2, pt_prev, skb->dev);
1981 pt_prev = ptype;
1982 continue;
1983 }
1984
1985
1986 skb2 = skb_clone(skb, GFP_ATOMIC);
1987 if (!skb2)
1988 goto out_unlock;
1989
1990 net_timestamp_set(skb2);
1991
1992
1993
1994
1995
1996 skb_reset_mac_header(skb2);
1997
1998 if (skb_network_header(skb2) < skb2->data ||
1999 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2000 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2001 ntohs(skb2->protocol),
2002 dev->name);
2003 skb_reset_network_header(skb2);
2004 }
2005
2006 skb2->transport_header = skb2->network_header;
2007 skb2->pkt_type = PACKET_OUTGOING;
2008 pt_prev = ptype;
2009 }
2010
2011 if (ptype_list == &ptype_all) {
2012 ptype_list = &dev->ptype_all;
2013 goto again;
2014 }
2015out_unlock:
2016 if (pt_prev) {
2017 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2018 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2019 else
2020 kfree_skb(skb2);
2021 }
2022 rcu_read_unlock();
2023}
2024EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2040{
2041 int i;
2042 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2043
2044
2045 if (tc->offset + tc->count > txq) {
2046 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2047 dev->num_tc = 0;
2048 return;
2049 }
2050
2051
2052 for (i = 1; i < TC_BITMASK + 1; i++) {
2053 int q = netdev_get_prio_tc_map(dev, i);
2054
2055 tc = &dev->tc_to_txq[q];
2056 if (tc->offset + tc->count > txq) {
2057 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2058 i, q);
2059 netdev_set_prio_tc_map(dev, i, 0);
2060 }
2061 }
2062}
2063
2064int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2065{
2066 if (dev->num_tc) {
2067 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2068 int i;
2069
2070 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2071 if ((txq - tc->offset) < tc->count)
2072 return i;
2073 }
2074
2075 return -1;
2076 }
2077
2078 return 0;
2079}
2080EXPORT_SYMBOL(netdev_txq_to_tc);
2081
2082#ifdef CONFIG_XPS
2083static DEFINE_MUTEX(xps_map_mutex);
2084#define xmap_dereference(P) \
2085 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2086
2087static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2088 int tci, u16 index)
2089{
2090 struct xps_map *map = NULL;
2091 int pos;
2092
2093 if (dev_maps)
2094 map = xmap_dereference(dev_maps->cpu_map[tci]);
2095 if (!map)
2096 return false;
2097
2098 for (pos = map->len; pos--;) {
2099 if (map->queues[pos] != index)
2100 continue;
2101
2102 if (map->len > 1) {
2103 map->queues[pos] = map->queues[--map->len];
2104 break;
2105 }
2106
2107 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2108 kfree_rcu(map, rcu);
2109 return false;
2110 }
2111
2112 return true;
2113}
2114
2115static bool remove_xps_queue_cpu(struct net_device *dev,
2116 struct xps_dev_maps *dev_maps,
2117 int cpu, u16 offset, u16 count)
2118{
2119 int num_tc = dev->num_tc ? : 1;
2120 bool active = false;
2121 int tci;
2122
2123 for (tci = cpu * num_tc; num_tc--; tci++) {
2124 int i, j;
2125
2126 for (i = count, j = offset; i--; j++) {
2127 if (!remove_xps_queue(dev_maps, tci, j))
2128 break;
2129 }
2130
2131 active |= i < 0;
2132 }
2133
2134 return active;
2135}
2136
2137static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2138 u16 count)
2139{
2140 struct xps_dev_maps *dev_maps;
2141 int cpu, i;
2142 bool active = false;
2143
2144 mutex_lock(&xps_map_mutex);
2145 dev_maps = xmap_dereference(dev->xps_maps);
2146
2147 if (!dev_maps)
2148 goto out_no_maps;
2149
2150 for_each_possible_cpu(cpu)
2151 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2152 offset, count);
2153
2154 if (!active) {
2155 RCU_INIT_POINTER(dev->xps_maps, NULL);
2156 kfree_rcu(dev_maps, rcu);
2157 }
2158
2159 for (i = offset + (count - 1); count--; i--)
2160 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2161 NUMA_NO_NODE);
2162
2163out_no_maps:
2164 mutex_unlock(&xps_map_mutex);
2165}
2166
2167static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2168{
2169 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2170}
2171
2172static struct xps_map *expand_xps_map(struct xps_map *map,
2173 int cpu, u16 index)
2174{
2175 struct xps_map *new_map;
2176 int alloc_len = XPS_MIN_MAP_ALLOC;
2177 int i, pos;
2178
2179 for (pos = 0; map && pos < map->len; pos++) {
2180 if (map->queues[pos] != index)
2181 continue;
2182 return map;
2183 }
2184
2185
2186 if (map) {
2187 if (pos < map->alloc_len)
2188 return map;
2189
2190 alloc_len = map->alloc_len * 2;
2191 }
2192
2193
2194 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2195 cpu_to_node(cpu));
2196 if (!new_map)
2197 return NULL;
2198
2199 for (i = 0; i < pos; i++)
2200 new_map->queues[i] = map->queues[i];
2201 new_map->alloc_len = alloc_len;
2202 new_map->len = pos;
2203
2204 return new_map;
2205}
2206
2207int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2208 u16 index)
2209{
2210 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2211 int i, cpu, tci, numa_node_id = -2;
2212 int maps_sz, num_tc = 1, tc = 0;
2213 struct xps_map *map, *new_map;
2214 bool active = false;
2215
2216 if (dev->num_tc) {
2217 num_tc = dev->num_tc;
2218 tc = netdev_txq_to_tc(dev, index);
2219 if (tc < 0)
2220 return -EINVAL;
2221 }
2222
2223 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2224 if (maps_sz < L1_CACHE_BYTES)
2225 maps_sz = L1_CACHE_BYTES;
2226
2227 mutex_lock(&xps_map_mutex);
2228
2229 dev_maps = xmap_dereference(dev->xps_maps);
2230
2231
2232 for_each_cpu_and(cpu, cpu_online_mask, mask) {
2233 if (!new_dev_maps)
2234 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2235 if (!new_dev_maps) {
2236 mutex_unlock(&xps_map_mutex);
2237 return -ENOMEM;
2238 }
2239
2240 tci = cpu * num_tc + tc;
2241 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2242 NULL;
2243
2244 map = expand_xps_map(map, cpu, index);
2245 if (!map)
2246 goto error;
2247
2248 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2249 }
2250
2251 if (!new_dev_maps)
2252 goto out_no_new_maps;
2253
2254 for_each_possible_cpu(cpu) {
2255
2256 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2257
2258 map = xmap_dereference(dev_maps->cpu_map[tci]);
2259 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2260 }
2261
2262
2263
2264
2265 tci = cpu * num_tc + tc;
2266
2267 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2268
2269 int pos = 0;
2270
2271 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2272 while ((pos < map->len) && (map->queues[pos] != index))
2273 pos++;
2274
2275 if (pos == map->len)
2276 map->queues[map->len++] = index;
2277#ifdef CONFIG_NUMA
2278 if (numa_node_id == -2)
2279 numa_node_id = cpu_to_node(cpu);
2280 else if (numa_node_id != cpu_to_node(cpu))
2281 numa_node_id = -1;
2282#endif
2283 } else if (dev_maps) {
2284
2285 map = xmap_dereference(dev_maps->cpu_map[tci]);
2286 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2287 }
2288
2289
2290 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2291
2292 map = xmap_dereference(dev_maps->cpu_map[tci]);
2293 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2294 }
2295 }
2296
2297 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2298
2299
2300 if (!dev_maps)
2301 goto out_no_old_maps;
2302
2303 for_each_possible_cpu(cpu) {
2304 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2305 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2306 map = xmap_dereference(dev_maps->cpu_map[tci]);
2307 if (map && map != new_map)
2308 kfree_rcu(map, rcu);
2309 }
2310 }
2311
2312 kfree_rcu(dev_maps, rcu);
2313
2314out_no_old_maps:
2315 dev_maps = new_dev_maps;
2316 active = true;
2317
2318out_no_new_maps:
2319
2320 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2321 (numa_node_id >= 0) ? numa_node_id :
2322 NUMA_NO_NODE);
2323
2324 if (!dev_maps)
2325 goto out_no_maps;
2326
2327
2328 for_each_possible_cpu(cpu) {
2329 for (i = tc, tci = cpu * num_tc; i--; tci++)
2330 active |= remove_xps_queue(dev_maps, tci, index);
2331 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2332 active |= remove_xps_queue(dev_maps, tci, index);
2333 for (i = num_tc - tc, tci++; --i; tci++)
2334 active |= remove_xps_queue(dev_maps, tci, index);
2335 }
2336
2337
2338 if (!active) {
2339 RCU_INIT_POINTER(dev->xps_maps, NULL);
2340 kfree_rcu(dev_maps, rcu);
2341 }
2342
2343out_no_maps:
2344 mutex_unlock(&xps_map_mutex);
2345
2346 return 0;
2347error:
2348
2349 for_each_possible_cpu(cpu) {
2350 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2351 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2352 map = dev_maps ?
2353 xmap_dereference(dev_maps->cpu_map[tci]) :
2354 NULL;
2355 if (new_map && new_map != map)
2356 kfree(new_map);
2357 }
2358 }
2359
2360 mutex_unlock(&xps_map_mutex);
2361
2362 kfree(new_dev_maps);
2363 return -ENOMEM;
2364}
2365EXPORT_SYMBOL(netif_set_xps_queue);
2366
2367#endif
2368void netdev_reset_tc(struct net_device *dev)
2369{
2370#ifdef CONFIG_XPS
2371 netif_reset_xps_queues_gt(dev, 0);
2372#endif
2373 dev->num_tc = 0;
2374 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2375 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2376}
2377EXPORT_SYMBOL(netdev_reset_tc);
2378
2379int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2380{
2381 if (tc >= dev->num_tc)
2382 return -EINVAL;
2383
2384#ifdef CONFIG_XPS
2385 netif_reset_xps_queues(dev, offset, count);
2386#endif
2387 dev->tc_to_txq[tc].count = count;
2388 dev->tc_to_txq[tc].offset = offset;
2389 return 0;
2390}
2391EXPORT_SYMBOL(netdev_set_tc_queue);
2392
2393int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2394{
2395 if (num_tc > TC_MAX_QUEUE)
2396 return -EINVAL;
2397
2398#ifdef CONFIG_XPS
2399 netif_reset_xps_queues_gt(dev, 0);
2400#endif
2401 dev->num_tc = num_tc;
2402 return 0;
2403}
2404EXPORT_SYMBOL(netdev_set_num_tc);
2405
2406
2407
2408
2409
2410int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2411{
2412 bool disabling;
2413 int rc;
2414
2415 disabling = txq < dev->real_num_tx_queues;
2416
2417 if (txq < 1 || txq > dev->num_tx_queues)
2418 return -EINVAL;
2419
2420 if (dev->reg_state == NETREG_REGISTERED ||
2421 dev->reg_state == NETREG_UNREGISTERING) {
2422 ASSERT_RTNL();
2423
2424 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2425 txq);
2426 if (rc)
2427 return rc;
2428
2429 if (dev->num_tc)
2430 netif_setup_tc(dev, txq);
2431
2432 dev->real_num_tx_queues = txq;
2433
2434 if (disabling) {
2435 synchronize_net();
2436 qdisc_reset_all_tx_gt(dev, txq);
2437#ifdef CONFIG_XPS
2438 netif_reset_xps_queues_gt(dev, txq);
2439#endif
2440 }
2441 } else {
2442 dev->real_num_tx_queues = txq;
2443 }
2444
2445 return 0;
2446}
2447EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2448
2449#ifdef CONFIG_SYSFS
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2461{
2462 int rc;
2463
2464 if (rxq < 1 || rxq > dev->num_rx_queues)
2465 return -EINVAL;
2466
2467 if (dev->reg_state == NETREG_REGISTERED) {
2468 ASSERT_RTNL();
2469
2470 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2471 rxq);
2472 if (rc)
2473 return rc;
2474 }
2475
2476 dev->real_num_rx_queues = rxq;
2477 return 0;
2478}
2479EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2480#endif
2481
2482
2483
2484
2485
2486
2487
2488int netif_get_num_default_rss_queues(void)
2489{
2490 return is_kdump_kernel() ?
2491 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2492}
2493EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2494
2495static void __netif_reschedule(struct Qdisc *q)
2496{
2497 struct softnet_data *sd;
2498 unsigned long flags;
2499
2500 local_irq_save(flags);
2501 sd = this_cpu_ptr(&softnet_data);
2502 q->next_sched = NULL;
2503 *sd->output_queue_tailp = q;
2504 sd->output_queue_tailp = &q->next_sched;
2505 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2506 local_irq_restore(flags);
2507}
2508
2509void __netif_schedule(struct Qdisc *q)
2510{
2511 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2512 __netif_reschedule(q);
2513}
2514EXPORT_SYMBOL(__netif_schedule);
2515
2516struct dev_kfree_skb_cb {
2517 enum skb_free_reason reason;
2518};
2519
2520static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2521{
2522 return (struct dev_kfree_skb_cb *)skb->cb;
2523}
2524
2525void netif_schedule_queue(struct netdev_queue *txq)
2526{
2527 rcu_read_lock();
2528 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2529 struct Qdisc *q = rcu_dereference(txq->qdisc);
2530
2531 __netif_schedule(q);
2532 }
2533 rcu_read_unlock();
2534}
2535EXPORT_SYMBOL(netif_schedule_queue);
2536
2537void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2538{
2539 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2540 struct Qdisc *q;
2541
2542 rcu_read_lock();
2543 q = rcu_dereference(dev_queue->qdisc);
2544 __netif_schedule(q);
2545 rcu_read_unlock();
2546 }
2547}
2548EXPORT_SYMBOL(netif_tx_wake_queue);
2549
2550void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2551{
2552 unsigned long flags;
2553
2554 if (unlikely(!skb))
2555 return;
2556
2557 if (likely(refcount_read(&skb->users) == 1)) {
2558 smp_rmb();
2559 refcount_set(&skb->users, 0);
2560 } else if (likely(!refcount_dec_and_test(&skb->users))) {
2561 return;
2562 }
2563 get_kfree_skb_cb(skb)->reason = reason;
2564 local_irq_save(flags);
2565 skb->next = __this_cpu_read(softnet_data.completion_queue);
2566 __this_cpu_write(softnet_data.completion_queue, skb);
2567 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2568 local_irq_restore(flags);
2569}
2570EXPORT_SYMBOL(__dev_kfree_skb_irq);
2571
2572void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2573{
2574 if (in_irq() || irqs_disabled())
2575 __dev_kfree_skb_irq(skb, reason);
2576 else
2577 dev_kfree_skb(skb);
2578}
2579EXPORT_SYMBOL(__dev_kfree_skb_any);
2580
2581
2582
2583
2584
2585
2586
2587
2588void netif_device_detach(struct net_device *dev)
2589{
2590 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2591 netif_running(dev)) {
2592 netif_tx_stop_all_queues(dev);
2593 }
2594}
2595EXPORT_SYMBOL(netif_device_detach);
2596
2597
2598
2599
2600
2601
2602
2603void netif_device_attach(struct net_device *dev)
2604{
2605 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2606 netif_running(dev)) {
2607 netif_tx_wake_all_queues(dev);
2608 __netdev_watchdog_up(dev);
2609 }
2610}
2611EXPORT_SYMBOL(netif_device_attach);
2612
2613
2614
2615
2616
2617u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2618 unsigned int num_tx_queues)
2619{
2620 u32 hash;
2621 u16 qoffset = 0;
2622 u16 qcount = num_tx_queues;
2623
2624 if (skb_rx_queue_recorded(skb)) {
2625 hash = skb_get_rx_queue(skb);
2626 while (unlikely(hash >= num_tx_queues))
2627 hash -= num_tx_queues;
2628 return hash;
2629 }
2630
2631 if (dev->num_tc) {
2632 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2633
2634 qoffset = dev->tc_to_txq[tc].offset;
2635 qcount = dev->tc_to_txq[tc].count;
2636 }
2637
2638 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2639}
2640EXPORT_SYMBOL(__skb_tx_hash);
2641
2642static void skb_warn_bad_offload(const struct sk_buff *skb)
2643{
2644 static const netdev_features_t null_features;
2645 struct net_device *dev = skb->dev;
2646 const char *name = "";
2647
2648 if (!net_ratelimit())
2649 return;
2650
2651 if (dev) {
2652 if (dev->dev.parent)
2653 name = dev_driver_string(dev->dev.parent);
2654 else
2655 name = netdev_name(dev);
2656 }
2657 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2658 "gso_type=%d ip_summed=%d\n",
2659 name, dev ? &dev->features : &null_features,
2660 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2661 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2662 skb_shinfo(skb)->gso_type, skb->ip_summed);
2663}
2664
2665
2666
2667
2668
2669int skb_checksum_help(struct sk_buff *skb)
2670{
2671 __wsum csum;
2672 int ret = 0, offset;
2673
2674 if (skb->ip_summed == CHECKSUM_COMPLETE)
2675 goto out_set_summed;
2676
2677 if (unlikely(skb_shinfo(skb)->gso_size)) {
2678 skb_warn_bad_offload(skb);
2679 return -EINVAL;
2680 }
2681
2682
2683
2684
2685 if (skb_has_shared_frag(skb)) {
2686 ret = __skb_linearize(skb);
2687 if (ret)
2688 goto out;
2689 }
2690
2691 offset = skb_checksum_start_offset(skb);
2692 BUG_ON(offset >= skb_headlen(skb));
2693 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2694
2695 offset += skb->csum_offset;
2696 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2697
2698 if (skb_cloned(skb) &&
2699 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2700 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2701 if (ret)
2702 goto out;
2703 }
2704
2705 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2706out_set_summed:
2707 skb->ip_summed = CHECKSUM_NONE;
2708out:
2709 return ret;
2710}
2711EXPORT_SYMBOL(skb_checksum_help);
2712
2713int skb_crc32c_csum_help(struct sk_buff *skb)
2714{
2715 __le32 crc32c_csum;
2716 int ret = 0, offset, start;
2717
2718 if (skb->ip_summed != CHECKSUM_PARTIAL)
2719 goto out;
2720
2721 if (unlikely(skb_is_gso(skb)))
2722 goto out;
2723
2724
2725
2726
2727 if (unlikely(skb_has_shared_frag(skb))) {
2728 ret = __skb_linearize(skb);
2729 if (ret)
2730 goto out;
2731 }
2732 start = skb_checksum_start_offset(skb);
2733 offset = start + offsetof(struct sctphdr, checksum);
2734 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2735 ret = -EINVAL;
2736 goto out;
2737 }
2738 if (skb_cloned(skb) &&
2739 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2740 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2741 if (ret)
2742 goto out;
2743 }
2744 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2745 skb->len - start, ~(__u32)0,
2746 crc32c_csum_stub));
2747 *(__le32 *)(skb->data + offset) = crc32c_csum;
2748 skb->ip_summed = CHECKSUM_NONE;
2749 skb->csum_not_inet = 0;
2750out:
2751 return ret;
2752}
2753
2754__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2755{
2756 __be16 type = skb->protocol;
2757
2758
2759 if (type == htons(ETH_P_TEB)) {
2760 struct ethhdr *eth;
2761
2762 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2763 return 0;
2764
2765 eth = (struct ethhdr *)skb->data;
2766 type = eth->h_proto;
2767 }
2768
2769 return __vlan_get_protocol(skb, type, depth);
2770}
2771
2772
2773
2774
2775
2776
2777struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2778 netdev_features_t features)
2779{
2780 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2781 struct packet_offload *ptype;
2782 int vlan_depth = skb->mac_len;
2783 __be16 type = skb_network_protocol(skb, &vlan_depth);
2784
2785 if (unlikely(!type))
2786 return ERR_PTR(-EINVAL);
2787
2788 __skb_pull(skb, vlan_depth);
2789
2790 rcu_read_lock();
2791 list_for_each_entry_rcu(ptype, &offload_base, list) {
2792 if (ptype->type == type && ptype->callbacks.gso_segment) {
2793 segs = ptype->callbacks.gso_segment(skb, features);
2794 break;
2795 }
2796 }
2797 rcu_read_unlock();
2798
2799 __skb_push(skb, skb->data - skb_mac_header(skb));
2800
2801 return segs;
2802}
2803EXPORT_SYMBOL(skb_mac_gso_segment);
2804
2805
2806
2807
2808static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2809{
2810 if (tx_path)
2811 return skb->ip_summed != CHECKSUM_PARTIAL &&
2812 skb->ip_summed != CHECKSUM_UNNECESSARY;
2813
2814 return skb->ip_summed == CHECKSUM_NONE;
2815}
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2831 netdev_features_t features, bool tx_path)
2832{
2833 struct sk_buff *segs;
2834
2835 if (unlikely(skb_needs_check(skb, tx_path))) {
2836 int err;
2837
2838
2839 err = skb_cow_head(skb, 0);
2840 if (err < 0)
2841 return ERR_PTR(err);
2842 }
2843
2844
2845
2846
2847
2848 if (features & NETIF_F_GSO_PARTIAL) {
2849 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2850 struct net_device *dev = skb->dev;
2851
2852 partial_features |= dev->features & dev->gso_partial_features;
2853 if (!skb_gso_ok(skb, features | partial_features))
2854 features &= ~NETIF_F_GSO_PARTIAL;
2855 }
2856
2857 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2858 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2859
2860 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2861 SKB_GSO_CB(skb)->encap_level = 0;
2862
2863 skb_reset_mac_header(skb);
2864 skb_reset_mac_len(skb);
2865
2866 segs = skb_mac_gso_segment(skb, features);
2867
2868 if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
2869 skb_warn_bad_offload(skb);
2870
2871 return segs;
2872}
2873EXPORT_SYMBOL(__skb_gso_segment);
2874
2875
2876#ifdef CONFIG_BUG
2877void netdev_rx_csum_fault(struct net_device *dev)
2878{
2879 if (net_ratelimit()) {
2880 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2881 dump_stack();
2882 }
2883}
2884EXPORT_SYMBOL(netdev_rx_csum_fault);
2885#endif
2886
2887
2888
2889
2890
2891
2892static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2893{
2894#ifdef CONFIG_HIGHMEM
2895 int i;
2896
2897 if (!(dev->features & NETIF_F_HIGHDMA)) {
2898 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2899 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2900
2901 if (PageHighMem(skb_frag_page(frag)))
2902 return 1;
2903 }
2904 }
2905
2906 if (PCI_DMA_BUS_IS_PHYS) {
2907 struct device *pdev = dev->dev.parent;
2908
2909 if (!pdev)
2910 return 0;
2911 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2912 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2913 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2914
2915 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2916 return 1;
2917 }
2918 }
2919#endif
2920 return 0;
2921}
2922
2923
2924
2925
2926#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2927static netdev_features_t net_mpls_features(struct sk_buff *skb,
2928 netdev_features_t features,
2929 __be16 type)
2930{
2931 if (eth_p_mpls(type))
2932 features &= skb->dev->mpls_features;
2933
2934 return features;
2935}
2936#else
2937static netdev_features_t net_mpls_features(struct sk_buff *skb,
2938 netdev_features_t features,
2939 __be16 type)
2940{
2941 return features;
2942}
2943#endif
2944
2945static netdev_features_t harmonize_features(struct sk_buff *skb,
2946 netdev_features_t features)
2947{
2948 int tmp;
2949 __be16 type;
2950
2951 type = skb_network_protocol(skb, &tmp);
2952 features = net_mpls_features(skb, features, type);
2953
2954 if (skb->ip_summed != CHECKSUM_NONE &&
2955 !can_checksum_protocol(features, type)) {
2956 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2957 }
2958 if (illegal_highdma(skb->dev, skb))
2959 features &= ~NETIF_F_SG;
2960
2961 return features;
2962}
2963
2964netdev_features_t passthru_features_check(struct sk_buff *skb,
2965 struct net_device *dev,
2966 netdev_features_t features)
2967{
2968 return features;
2969}
2970EXPORT_SYMBOL(passthru_features_check);
2971
2972static netdev_features_t dflt_features_check(struct sk_buff *skb,
2973 struct net_device *dev,
2974 netdev_features_t features)
2975{
2976 return vlan_features_check(skb, features);
2977}
2978
2979static netdev_features_t gso_features_check(const struct sk_buff *skb,
2980 struct net_device *dev,
2981 netdev_features_t features)
2982{
2983 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2984
2985 if (gso_segs > dev->gso_max_segs)
2986 return features & ~NETIF_F_GSO_MASK;
2987
2988
2989
2990
2991
2992
2993
2994 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2995 features &= ~dev->gso_partial_features;
2996
2997
2998
2999
3000 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3001 struct iphdr *iph = skb->encapsulation ?
3002 inner_ip_hdr(skb) : ip_hdr(skb);
3003
3004 if (!(iph->frag_off & htons(IP_DF)))
3005 features &= ~NETIF_F_TSO_MANGLEID;
3006 }
3007
3008 return features;
3009}
3010
3011netdev_features_t netif_skb_features(struct sk_buff *skb)
3012{
3013 struct net_device *dev = skb->dev;
3014 netdev_features_t features = dev->features;
3015
3016 if (skb_is_gso(skb))
3017 features = gso_features_check(skb, dev, features);
3018
3019
3020
3021
3022
3023 if (skb->encapsulation)
3024 features &= dev->hw_enc_features;
3025
3026 if (skb_vlan_tagged(skb))
3027 features = netdev_intersect_features(features,
3028 dev->vlan_features |
3029 NETIF_F_HW_VLAN_CTAG_TX |
3030 NETIF_F_HW_VLAN_STAG_TX);
3031
3032 if (dev->netdev_ops->ndo_features_check)
3033 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3034 features);
3035 else
3036 features &= dflt_features_check(skb, dev, features);
3037
3038 return harmonize_features(skb, features);
3039}
3040EXPORT_SYMBOL(netif_skb_features);
3041
3042static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3043 struct netdev_queue *txq, bool more)
3044{
3045 unsigned int len;
3046 int rc;
3047
3048 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
3049 dev_queue_xmit_nit(skb, dev);
3050
3051 len = skb->len;
3052 trace_net_dev_start_xmit(skb, dev);
3053 rc = netdev_start_xmit(skb, dev, txq, more);
3054 trace_net_dev_xmit(skb, rc, dev, len);
3055
3056 return rc;
3057}
3058
3059struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3060 struct netdev_queue *txq, int *ret)
3061{
3062 struct sk_buff *skb = first;
3063 int rc = NETDEV_TX_OK;
3064
3065 while (skb) {
3066 struct sk_buff *next = skb->next;
3067
3068 skb->next = NULL;
3069 rc = xmit_one(skb, dev, txq, next != NULL);
3070 if (unlikely(!dev_xmit_complete(rc))) {
3071 skb->next = next;
3072 goto out;
3073 }
3074
3075 skb = next;
3076 if (netif_xmit_stopped(txq) && skb) {
3077 rc = NETDEV_TX_BUSY;
3078 break;
3079 }
3080 }
3081
3082out:
3083 *ret = rc;
3084 return skb;
3085}
3086
3087static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3088 netdev_features_t features)
3089{
3090 if (skb_vlan_tag_present(skb) &&
3091 !vlan_hw_offload_capable(features, skb->vlan_proto))
3092 skb = __vlan_hwaccel_push_inside(skb);
3093 return skb;
3094}
3095
3096int skb_csum_hwoffload_help(struct sk_buff *skb,
3097 const netdev_features_t features)
3098{
3099 if (unlikely(skb->csum_not_inet))
3100 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3101 skb_crc32c_csum_help(skb);
3102
3103 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3104}
3105EXPORT_SYMBOL(skb_csum_hwoffload_help);
3106
3107static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3108{
3109 netdev_features_t features;
3110
3111 features = netif_skb_features(skb);
3112 skb = validate_xmit_vlan(skb, features);
3113 if (unlikely(!skb))
3114 goto out_null;
3115
3116 if (netif_needs_gso(skb, features)) {
3117 struct sk_buff *segs;
3118
3119 segs = skb_gso_segment(skb, features);
3120 if (IS_ERR(segs)) {
3121 goto out_kfree_skb;
3122 } else if (segs) {
3123 consume_skb(skb);
3124 skb = segs;
3125 }
3126 } else {
3127 if (skb_needs_linearize(skb, features) &&
3128 __skb_linearize(skb))
3129 goto out_kfree_skb;
3130
3131
3132
3133
3134
3135 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3136 if (skb->encapsulation)
3137 skb_set_inner_transport_header(skb,
3138 skb_checksum_start_offset(skb));
3139 else
3140 skb_set_transport_header(skb,
3141 skb_checksum_start_offset(skb));
3142 if (skb_csum_hwoffload_help(skb, features))
3143 goto out_kfree_skb;
3144 }
3145 }
3146
3147 skb = validate_xmit_xfrm(skb, features, again);
3148
3149 return skb;
3150
3151out_kfree_skb:
3152 kfree_skb(skb);
3153out_null:
3154 atomic_long_inc(&dev->tx_dropped);
3155 return NULL;
3156}
3157
3158struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3159{
3160 struct sk_buff *next, *head = NULL, *tail;
3161
3162 for (; skb != NULL; skb = next) {
3163 next = skb->next;
3164 skb->next = NULL;
3165
3166
3167 skb->prev = skb;
3168
3169 skb = validate_xmit_skb(skb, dev, again);
3170 if (!skb)
3171 continue;
3172
3173 if (!head)
3174 head = skb;
3175 else
3176 tail->next = skb;
3177
3178
3179
3180 tail = skb->prev;
3181 }
3182 return head;
3183}
3184EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3185
3186static void qdisc_pkt_len_init(struct sk_buff *skb)
3187{
3188 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3189
3190 qdisc_skb_cb(skb)->pkt_len = skb->len;
3191
3192
3193
3194
3195 if (shinfo->gso_size) {
3196 unsigned int hdr_len;
3197 u16 gso_segs = shinfo->gso_segs;
3198
3199
3200 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3201
3202
3203 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3204 const struct tcphdr *th;
3205 struct tcphdr _tcphdr;
3206
3207 th = skb_header_pointer(skb, skb_transport_offset(skb),
3208 sizeof(_tcphdr), &_tcphdr);
3209 if (likely(th))
3210 hdr_len += __tcp_hdrlen(th);
3211 } else {
3212 struct udphdr _udphdr;
3213
3214 if (skb_header_pointer(skb, skb_transport_offset(skb),
3215 sizeof(_udphdr), &_udphdr))
3216 hdr_len += sizeof(struct udphdr);
3217 }
3218
3219 if (shinfo->gso_type & SKB_GSO_DODGY)
3220 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3221 shinfo->gso_size);
3222
3223 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3224 }
3225}
3226
3227static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3228 struct net_device *dev,
3229 struct netdev_queue *txq)
3230{
3231 spinlock_t *root_lock = qdisc_lock(q);
3232 struct sk_buff *to_free = NULL;
3233 bool contended;
3234 int rc;
3235
3236 qdisc_calculate_pkt_len(skb, q);
3237
3238 if (q->flags & TCQ_F_NOLOCK) {
3239 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3240 __qdisc_drop(skb, &to_free);
3241 rc = NET_XMIT_DROP;
3242 } else {
3243 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3244 __qdisc_run(q);
3245 }
3246
3247 if (unlikely(to_free))
3248 kfree_skb_list(to_free);
3249 return rc;
3250 }
3251
3252
3253
3254
3255
3256
3257
3258 contended = qdisc_is_running(q);
3259 if (unlikely(contended))
3260 spin_lock(&q->busylock);
3261
3262 spin_lock(root_lock);
3263 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3264 __qdisc_drop(skb, &to_free);
3265 rc = NET_XMIT_DROP;
3266 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3267 qdisc_run_begin(q)) {
3268
3269
3270
3271
3272
3273
3274 qdisc_bstats_update(q, skb);
3275
3276 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3277 if (unlikely(contended)) {
3278 spin_unlock(&q->busylock);
3279 contended = false;
3280 }
3281 __qdisc_run(q);
3282 }
3283
3284 qdisc_run_end(q);
3285 rc = NET_XMIT_SUCCESS;
3286 } else {
3287 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3288 if (qdisc_run_begin(q)) {
3289 if (unlikely(contended)) {
3290 spin_unlock(&q->busylock);
3291 contended = false;
3292 }
3293 __qdisc_run(q);
3294 qdisc_run_end(q);
3295 }
3296 }
3297 spin_unlock(root_lock);
3298 if (unlikely(to_free))
3299 kfree_skb_list(to_free);
3300 if (unlikely(contended))
3301 spin_unlock(&q->busylock);
3302 return rc;
3303}
3304
3305#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3306static void skb_update_prio(struct sk_buff *skb)
3307{
3308 const struct netprio_map *map;
3309 const struct sock *sk;
3310 unsigned int prioidx;
3311
3312 if (skb->priority)
3313 return;
3314 map = rcu_dereference_bh(skb->dev->priomap);
3315 if (!map)
3316 return;
3317 sk = skb_to_full_sk(skb);
3318 if (!sk)
3319 return;
3320
3321 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3322
3323 if (prioidx < map->priomap_len)
3324 skb->priority = map->priomap[prioidx];
3325}
3326#else
3327#define skb_update_prio(skb)
3328#endif
3329
3330DEFINE_PER_CPU(int, xmit_recursion);
3331EXPORT_SYMBOL(xmit_recursion);
3332
3333
3334
3335
3336
3337
3338
3339int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3340{
3341 skb_reset_mac_header(skb);
3342 __skb_pull(skb, skb_network_offset(skb));
3343 skb->pkt_type = PACKET_LOOPBACK;
3344 skb->ip_summed = CHECKSUM_UNNECESSARY;
3345 WARN_ON(!skb_dst(skb));
3346 skb_dst_force(skb);
3347 netif_rx_ni(skb);
3348 return 0;
3349}
3350EXPORT_SYMBOL(dev_loopback_xmit);
3351
3352#ifdef CONFIG_NET_EGRESS
3353static struct sk_buff *
3354sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3355{
3356 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3357 struct tcf_result cl_res;
3358
3359 if (!miniq)
3360 return skb;
3361
3362
3363 mini_qdisc_bstats_cpu_update(miniq, skb);
3364
3365 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
3366 case TC_ACT_OK:
3367 case TC_ACT_RECLASSIFY:
3368 skb->tc_index = TC_H_MIN(cl_res.classid);
3369 break;
3370 case TC_ACT_SHOT:
3371 mini_qdisc_qstats_cpu_drop(miniq);
3372 *ret = NET_XMIT_DROP;
3373 kfree_skb(skb);
3374 return NULL;
3375 case TC_ACT_STOLEN:
3376 case TC_ACT_QUEUED:
3377 case TC_ACT_TRAP:
3378 *ret = NET_XMIT_SUCCESS;
3379 consume_skb(skb);
3380 return NULL;
3381 case TC_ACT_REDIRECT:
3382
3383 skb_do_redirect(skb);
3384 *ret = NET_XMIT_SUCCESS;
3385 return NULL;
3386 default:
3387 break;
3388 }
3389
3390 return skb;
3391}
3392#endif
3393
3394static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3395{
3396#ifdef CONFIG_XPS
3397 struct xps_dev_maps *dev_maps;
3398 struct xps_map *map;
3399 int queue_index = -1;
3400
3401 rcu_read_lock();
3402 dev_maps = rcu_dereference(dev->xps_maps);
3403 if (dev_maps) {
3404 unsigned int tci = skb->sender_cpu - 1;
3405
3406 if (dev->num_tc) {
3407 tci *= dev->num_tc;
3408 tci += netdev_get_prio_tc_map(dev, skb->priority);
3409 }
3410
3411 map = rcu_dereference(dev_maps->cpu_map[tci]);
3412 if (map) {
3413 if (map->len == 1)
3414 queue_index = map->queues[0];
3415 else
3416 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3417 map->len)];
3418 if (unlikely(queue_index >= dev->real_num_tx_queues))
3419 queue_index = -1;
3420 }
3421 }
3422 rcu_read_unlock();
3423
3424 return queue_index;
3425#else
3426 return -1;
3427#endif
3428}
3429
3430static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3431{
3432 struct sock *sk = skb->sk;
3433 int queue_index = sk_tx_queue_get(sk);
3434
3435 if (queue_index < 0 || skb->ooo_okay ||
3436 queue_index >= dev->real_num_tx_queues) {
3437 int new_index = get_xps_queue(dev, skb);
3438
3439 if (new_index < 0)
3440 new_index = skb_tx_hash(dev, skb);
3441
3442 if (queue_index != new_index && sk &&
3443 sk_fullsock(sk) &&
3444 rcu_access_pointer(sk->sk_dst_cache))
3445 sk_tx_queue_set(sk, new_index);
3446
3447 queue_index = new_index;
3448 }
3449
3450 return queue_index;
3451}
3452
3453struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3454 struct sk_buff *skb,
3455 void *accel_priv)
3456{
3457 int queue_index = 0;
3458
3459#ifdef CONFIG_XPS
3460 u32 sender_cpu = skb->sender_cpu - 1;
3461
3462 if (sender_cpu >= (u32)NR_CPUS)
3463 skb->sender_cpu = raw_smp_processor_id() + 1;
3464#endif
3465
3466 if (dev->real_num_tx_queues != 1) {
3467 const struct net_device_ops *ops = dev->netdev_ops;
3468
3469 if (ops->ndo_select_queue)
3470 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3471 __netdev_pick_tx);
3472 else
3473 queue_index = __netdev_pick_tx(dev, skb);
3474
3475 queue_index = netdev_cap_txqueue(dev, queue_index);
3476 }
3477
3478 skb_set_queue_mapping(skb, queue_index);
3479 return netdev_get_tx_queue(dev, queue_index);
3480}
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3509{
3510 struct net_device *dev = skb->dev;
3511 struct netdev_queue *txq;
3512 struct Qdisc *q;
3513 int rc = -ENOMEM;
3514 bool again = false;
3515
3516 skb_reset_mac_header(skb);
3517
3518 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3519 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3520
3521
3522
3523
3524 rcu_read_lock_bh();
3525
3526 skb_update_prio(skb);
3527
3528 qdisc_pkt_len_init(skb);
3529#ifdef CONFIG_NET_CLS_ACT
3530 skb->tc_at_ingress = 0;
3531# ifdef CONFIG_NET_EGRESS
3532 if (static_key_false(&egress_needed)) {
3533 skb = sch_handle_egress(skb, &rc, dev);
3534 if (!skb)
3535 goto out;
3536 }
3537# endif
3538#endif
3539
3540
3541
3542 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3543 skb_dst_drop(skb);
3544 else
3545 skb_dst_force(skb);
3546
3547 txq = netdev_pick_tx(dev, skb, accel_priv);
3548 q = rcu_dereference_bh(txq->qdisc);
3549
3550 trace_net_dev_queue(skb);
3551 if (q->enqueue) {
3552 rc = __dev_xmit_skb(skb, q, dev, txq);
3553 goto out;
3554 }
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568 if (dev->flags & IFF_UP) {
3569 int cpu = smp_processor_id();
3570
3571 if (txq->xmit_lock_owner != cpu) {
3572 if (unlikely(__this_cpu_read(xmit_recursion) >
3573 XMIT_RECURSION_LIMIT))
3574 goto recursion_alert;
3575
3576 skb = validate_xmit_skb(skb, dev, &again);
3577 if (!skb)
3578 goto out;
3579
3580 HARD_TX_LOCK(dev, txq, cpu);
3581
3582 if (!netif_xmit_stopped(txq)) {
3583 __this_cpu_inc(xmit_recursion);
3584 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3585 __this_cpu_dec(xmit_recursion);
3586 if (dev_xmit_complete(rc)) {
3587 HARD_TX_UNLOCK(dev, txq);
3588 goto out;
3589 }
3590 }
3591 HARD_TX_UNLOCK(dev, txq);
3592 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3593 dev->name);
3594 } else {
3595
3596
3597
3598recursion_alert:
3599 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3600 dev->name);
3601 }
3602 }
3603
3604 rc = -ENETDOWN;
3605 rcu_read_unlock_bh();
3606
3607 atomic_long_inc(&dev->tx_dropped);
3608 kfree_skb_list(skb);
3609 return rc;
3610out:
3611 rcu_read_unlock_bh();
3612 return rc;
3613}
3614
3615int dev_queue_xmit(struct sk_buff *skb)
3616{
3617 return __dev_queue_xmit(skb, NULL);
3618}
3619EXPORT_SYMBOL(dev_queue_xmit);
3620
3621int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3622{
3623 return __dev_queue_xmit(skb, accel_priv);
3624}
3625EXPORT_SYMBOL(dev_queue_xmit_accel);
3626
3627
3628
3629
3630
3631
3632int netdev_max_backlog __read_mostly = 1000;
3633EXPORT_SYMBOL(netdev_max_backlog);
3634
3635int netdev_tstamp_prequeue __read_mostly = 1;
3636int netdev_budget __read_mostly = 300;
3637unsigned int __read_mostly netdev_budget_usecs = 2000;
3638int weight_p __read_mostly = 64;
3639int dev_weight_rx_bias __read_mostly = 1;
3640int dev_weight_tx_bias __read_mostly = 1;
3641int dev_rx_weight __read_mostly = 64;
3642int dev_tx_weight __read_mostly = 64;
3643
3644
3645static inline void ____napi_schedule(struct softnet_data *sd,
3646 struct napi_struct *napi)
3647{
3648 list_add_tail(&napi->poll_list, &sd->poll_list);
3649 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3650}
3651
3652#ifdef CONFIG_RPS
3653
3654
3655struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3656EXPORT_SYMBOL(rps_sock_flow_table);
3657u32 rps_cpu_mask __read_mostly;
3658EXPORT_SYMBOL(rps_cpu_mask);
3659
3660struct static_key rps_needed __read_mostly;
3661EXPORT_SYMBOL(rps_needed);
3662struct static_key rfs_needed __read_mostly;
3663EXPORT_SYMBOL(rfs_needed);
3664
3665static struct rps_dev_flow *
3666set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3667 struct rps_dev_flow *rflow, u16 next_cpu)
3668{
3669 if (next_cpu < nr_cpu_ids) {
3670#ifdef CONFIG_RFS_ACCEL
3671 struct netdev_rx_queue *rxqueue;
3672 struct rps_dev_flow_table *flow_table;
3673 struct rps_dev_flow *old_rflow;
3674 u32 flow_id;
3675 u16 rxq_index;
3676 int rc;
3677
3678
3679 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3680 !(dev->features & NETIF_F_NTUPLE))
3681 goto out;
3682 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3683 if (rxq_index == skb_get_rx_queue(skb))
3684 goto out;
3685
3686 rxqueue = dev->_rx + rxq_index;
3687 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3688 if (!flow_table)
3689 goto out;
3690 flow_id = skb_get_hash(skb) & flow_table->mask;
3691 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3692 rxq_index, flow_id);
3693 if (rc < 0)
3694 goto out;
3695 old_rflow = rflow;
3696 rflow = &flow_table->flows[flow_id];
3697 rflow->filter = rc;
3698 if (old_rflow->filter == rflow->filter)
3699 old_rflow->filter = RPS_NO_FILTER;
3700 out:
3701#endif
3702 rflow->last_qtail =
3703 per_cpu(softnet_data, next_cpu).input_queue_head;
3704 }
3705
3706 rflow->cpu = next_cpu;
3707 return rflow;
3708}
3709
3710
3711
3712
3713
3714
3715static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3716 struct rps_dev_flow **rflowp)
3717{
3718 const struct rps_sock_flow_table *sock_flow_table;
3719 struct netdev_rx_queue *rxqueue = dev->_rx;
3720 struct rps_dev_flow_table *flow_table;
3721 struct rps_map *map;
3722 int cpu = -1;
3723 u32 tcpu;
3724 u32 hash;
3725
3726 if (skb_rx_queue_recorded(skb)) {
3727 u16 index = skb_get_rx_queue(skb);
3728
3729 if (unlikely(index >= dev->real_num_rx_queues)) {
3730 WARN_ONCE(dev->real_num_rx_queues > 1,
3731 "%s received packet on queue %u, but number "
3732 "of RX queues is %u\n",
3733 dev->name, index, dev->real_num_rx_queues);
3734 goto done;
3735 }
3736 rxqueue += index;
3737 }
3738
3739
3740
3741 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3742 map = rcu_dereference(rxqueue->rps_map);
3743 if (!flow_table && !map)
3744 goto done;
3745
3746 skb_reset_network_header(skb);
3747 hash = skb_get_hash(skb);
3748 if (!hash)
3749 goto done;
3750
3751 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3752 if (flow_table && sock_flow_table) {
3753 struct rps_dev_flow *rflow;
3754 u32 next_cpu;
3755 u32 ident;
3756
3757
3758 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3759 if ((ident ^ hash) & ~rps_cpu_mask)
3760 goto try_rps;
3761
3762 next_cpu = ident & rps_cpu_mask;
3763
3764
3765
3766
3767 rflow = &flow_table->flows[hash & flow_table->mask];
3768 tcpu = rflow->cpu;
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781 if (unlikely(tcpu != next_cpu) &&
3782 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3783 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3784 rflow->last_qtail)) >= 0)) {
3785 tcpu = next_cpu;
3786 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3787 }
3788
3789 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3790 *rflowp = rflow;
3791 cpu = tcpu;
3792 goto done;
3793 }
3794 }
3795
3796try_rps:
3797
3798 if (map) {
3799 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3800 if (cpu_online(tcpu)) {
3801 cpu = tcpu;
3802 goto done;
3803 }
3804 }
3805
3806done:
3807 return cpu;
3808}
3809
3810#ifdef CONFIG_RFS_ACCEL
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3824 u32 flow_id, u16 filter_id)
3825{
3826 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3827 struct rps_dev_flow_table *flow_table;
3828 struct rps_dev_flow *rflow;
3829 bool expire = true;
3830 unsigned int cpu;
3831
3832 rcu_read_lock();
3833 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3834 if (flow_table && flow_id <= flow_table->mask) {
3835 rflow = &flow_table->flows[flow_id];
3836 cpu = READ_ONCE(rflow->cpu);
3837 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3838 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3839 rflow->last_qtail) <
3840 (int)(10 * flow_table->mask)))
3841 expire = false;
3842 }
3843 rcu_read_unlock();
3844 return expire;
3845}
3846EXPORT_SYMBOL(rps_may_expire_flow);
3847
3848#endif
3849
3850
3851static void rps_trigger_softirq(void *data)
3852{
3853 struct softnet_data *sd = data;
3854
3855 ____napi_schedule(sd, &sd->backlog);
3856 sd->received_rps++;
3857}
3858
3859#endif
3860
3861
3862
3863
3864
3865
3866static int rps_ipi_queued(struct softnet_data *sd)
3867{
3868#ifdef CONFIG_RPS
3869 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3870
3871 if (sd != mysd) {
3872 sd->rps_ipi_next = mysd->rps_ipi_list;
3873 mysd->rps_ipi_list = sd;
3874
3875 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3876 return 1;
3877 }
3878#endif
3879 return 0;
3880}
3881
3882#ifdef CONFIG_NET_FLOW_LIMIT
3883int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3884#endif
3885
3886static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3887{
3888#ifdef CONFIG_NET_FLOW_LIMIT
3889 struct sd_flow_limit *fl;
3890 struct softnet_data *sd;
3891 unsigned int old_flow, new_flow;
3892
3893 if (qlen < (netdev_max_backlog >> 1))
3894 return false;
3895
3896 sd = this_cpu_ptr(&softnet_data);
3897
3898 rcu_read_lock();
3899 fl = rcu_dereference(sd->flow_limit);
3900 if (fl) {
3901 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3902 old_flow = fl->history[fl->history_head];
3903 fl->history[fl->history_head] = new_flow;
3904
3905 fl->history_head++;
3906 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3907
3908 if (likely(fl->buckets[old_flow]))
3909 fl->buckets[old_flow]--;
3910
3911 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3912 fl->count++;
3913 rcu_read_unlock();
3914 return true;
3915 }
3916 }
3917 rcu_read_unlock();
3918#endif
3919 return false;
3920}
3921
3922
3923
3924
3925
3926static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3927 unsigned int *qtail)
3928{
3929 struct softnet_data *sd;
3930 unsigned long flags;
3931 unsigned int qlen;
3932
3933 sd = &per_cpu(softnet_data, cpu);
3934
3935 local_irq_save(flags);
3936
3937 rps_lock(sd);
3938 if (!netif_running(skb->dev))
3939 goto drop;
3940 qlen = skb_queue_len(&sd->input_pkt_queue);
3941 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3942 if (qlen) {
3943enqueue:
3944 __skb_queue_tail(&sd->input_pkt_queue, skb);
3945 input_queue_tail_incr_save(sd, qtail);
3946 rps_unlock(sd);
3947 local_irq_restore(flags);
3948 return NET_RX_SUCCESS;
3949 }
3950
3951
3952
3953
3954 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3955 if (!rps_ipi_queued(sd))
3956 ____napi_schedule(sd, &sd->backlog);
3957 }
3958 goto enqueue;
3959 }
3960
3961drop:
3962 sd->dropped++;
3963 rps_unlock(sd);
3964
3965 local_irq_restore(flags);
3966
3967 atomic_long_inc(&skb->dev->rx_dropped);
3968 kfree_skb(skb);
3969 return NET_RX_DROP;
3970}
3971
3972static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
3973{
3974 struct net_device *dev = skb->dev;
3975 struct netdev_rx_queue *rxqueue;
3976
3977 rxqueue = dev->_rx;
3978
3979 if (skb_rx_queue_recorded(skb)) {
3980 u16 index = skb_get_rx_queue(skb);
3981
3982 if (unlikely(index >= dev->real_num_rx_queues)) {
3983 WARN_ONCE(dev->real_num_rx_queues > 1,
3984 "%s received packet on queue %u, but number "
3985 "of RX queues is %u\n",
3986 dev->name, index, dev->real_num_rx_queues);
3987
3988 return rxqueue;
3989 }
3990 rxqueue += index;
3991 }
3992 return rxqueue;
3993}
3994
3995static u32 netif_receive_generic_xdp(struct sk_buff *skb,
3996 struct bpf_prog *xdp_prog)
3997{
3998 struct netdev_rx_queue *rxqueue;
3999 u32 metalen, act = XDP_DROP;
4000 struct xdp_buff xdp;
4001 void *orig_data;
4002 int hlen, off;
4003 u32 mac_len;
4004
4005
4006
4007
4008 if (skb_cloned(skb))
4009 return XDP_PASS;
4010
4011
4012
4013
4014
4015 if (skb_is_nonlinear(skb) ||
4016 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4017 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4018 int troom = skb->tail + skb->data_len - skb->end;
4019
4020
4021
4022
4023 if (pskb_expand_head(skb,
4024 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4025 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4026 goto do_drop;
4027 if (skb_linearize(skb))
4028 goto do_drop;
4029 }
4030
4031
4032
4033
4034 mac_len = skb->data - skb_mac_header(skb);
4035 hlen = skb_headlen(skb) + mac_len;
4036 xdp.data = skb->data - mac_len;
4037 xdp.data_meta = xdp.data;
4038 xdp.data_end = xdp.data + hlen;
4039 xdp.data_hard_start = skb->data - skb_headroom(skb);
4040 orig_data = xdp.data;
4041
4042 rxqueue = netif_get_rxqueue(skb);
4043 xdp.rxq = &rxqueue->xdp_rxq;
4044
4045 act = bpf_prog_run_xdp(xdp_prog, &xdp);
4046
4047 off = xdp.data - orig_data;
4048 if (off > 0)
4049 __skb_pull(skb, off);
4050 else if (off < 0)
4051 __skb_push(skb, -off);
4052 skb->mac_header += off;
4053
4054 switch (act) {
4055 case XDP_REDIRECT:
4056 case XDP_TX:
4057 __skb_push(skb, mac_len);
4058 break;
4059 case XDP_PASS:
4060 metalen = xdp.data - xdp.data_meta;
4061 if (metalen)
4062 skb_metadata_set(skb, metalen);
4063 break;
4064 default:
4065 bpf_warn_invalid_xdp_action(act);
4066
4067 case XDP_ABORTED:
4068 trace_xdp_exception(skb->dev, xdp_prog, act);
4069
4070 case XDP_DROP:
4071 do_drop:
4072 kfree_skb(skb);
4073 break;
4074 }
4075
4076 return act;
4077}
4078
4079
4080
4081
4082void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4083{
4084 struct net_device *dev = skb->dev;
4085 struct netdev_queue *txq;
4086 bool free_skb = true;
4087 int cpu, rc;
4088
4089 txq = netdev_pick_tx(dev, skb, NULL);
4090 cpu = smp_processor_id();
4091 HARD_TX_LOCK(dev, txq, cpu);
4092 if (!netif_xmit_stopped(txq)) {
4093 rc = netdev_start_xmit(skb, dev, txq, 0);
4094 if (dev_xmit_complete(rc))
4095 free_skb = false;
4096 }
4097 HARD_TX_UNLOCK(dev, txq);
4098 if (free_skb) {
4099 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4100 kfree_skb(skb);
4101 }
4102}
4103EXPORT_SYMBOL_GPL(generic_xdp_tx);
4104
4105static struct static_key generic_xdp_needed __read_mostly;
4106
4107int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4108{
4109 if (xdp_prog) {
4110 u32 act = netif_receive_generic_xdp(skb, xdp_prog);
4111 int err;
4112
4113 if (act != XDP_PASS) {
4114 switch (act) {
4115 case XDP_REDIRECT:
4116 err = xdp_do_generic_redirect(skb->dev, skb,
4117 xdp_prog);
4118 if (err)
4119 goto out_redir;
4120
4121 case XDP_TX:
4122 generic_xdp_tx(skb, xdp_prog);
4123 break;
4124 }
4125 return XDP_DROP;
4126 }
4127 }
4128 return XDP_PASS;
4129out_redir:
4130 kfree_skb(skb);
4131 return XDP_DROP;
4132}
4133EXPORT_SYMBOL_GPL(do_xdp_generic);
4134
4135static int netif_rx_internal(struct sk_buff *skb)
4136{
4137 int ret;
4138
4139 net_timestamp_check(netdev_tstamp_prequeue, skb);
4140
4141 trace_netif_rx(skb);
4142
4143 if (static_key_false(&generic_xdp_needed)) {
4144 int ret;
4145
4146 preempt_disable();
4147 rcu_read_lock();
4148 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4149 rcu_read_unlock();
4150 preempt_enable();
4151
4152
4153
4154
4155
4156 if (ret != XDP_PASS)
4157 return NET_RX_SUCCESS;
4158 }
4159
4160#ifdef CONFIG_RPS
4161 if (static_key_false(&rps_needed)) {
4162 struct rps_dev_flow voidflow, *rflow = &voidflow;
4163 int cpu;
4164
4165 preempt_disable();
4166 rcu_read_lock();
4167
4168 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4169 if (cpu < 0)
4170 cpu = smp_processor_id();
4171
4172 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4173
4174 rcu_read_unlock();
4175 preempt_enable();
4176 } else
4177#endif
4178 {
4179 unsigned int qtail;
4180
4181 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4182 put_cpu();
4183 }
4184 return ret;
4185}
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202int netif_rx(struct sk_buff *skb)
4203{
4204 trace_netif_rx_entry(skb);
4205
4206 return netif_rx_internal(skb);
4207}
4208EXPORT_SYMBOL(netif_rx);
4209
4210int netif_rx_ni(struct sk_buff *skb)
4211{
4212 int err;
4213
4214 trace_netif_rx_ni_entry(skb);
4215
4216 preempt_disable();
4217 err = netif_rx_internal(skb);
4218 if (local_softirq_pending())
4219 do_softirq();
4220 preempt_enable();
4221
4222 return err;
4223}
4224EXPORT_SYMBOL(netif_rx_ni);
4225
4226static __latent_entropy void net_tx_action(struct softirq_action *h)
4227{
4228 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4229
4230 if (sd->completion_queue) {
4231 struct sk_buff *clist;
4232
4233 local_irq_disable();
4234 clist = sd->completion_queue;
4235 sd->completion_queue = NULL;
4236 local_irq_enable();
4237
4238 while (clist) {
4239 struct sk_buff *skb = clist;
4240
4241 clist = clist->next;
4242
4243 WARN_ON(refcount_read(&skb->users));
4244 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4245 trace_consume_skb(skb);
4246 else
4247 trace_kfree_skb(skb, net_tx_action);
4248
4249 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4250 __kfree_skb(skb);
4251 else
4252 __kfree_skb_defer(skb);
4253 }
4254
4255 __kfree_skb_flush();
4256 }
4257
4258 if (sd->output_queue) {
4259 struct Qdisc *head;
4260
4261 local_irq_disable();
4262 head = sd->output_queue;
4263 sd->output_queue = NULL;
4264 sd->output_queue_tailp = &sd->output_queue;
4265 local_irq_enable();
4266
4267 while (head) {
4268 struct Qdisc *q = head;
4269 spinlock_t *root_lock = NULL;
4270
4271 head = head->next_sched;
4272
4273 if (!(q->flags & TCQ_F_NOLOCK)) {
4274 root_lock = qdisc_lock(q);
4275 spin_lock(root_lock);
4276 }
4277
4278
4279
4280 smp_mb__before_atomic();
4281 clear_bit(__QDISC_STATE_SCHED, &q->state);
4282 qdisc_run(q);
4283 if (root_lock)
4284 spin_unlock(root_lock);
4285 }
4286 }
4287
4288 xfrm_dev_backlog(sd);
4289}
4290
4291#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4292
4293int (*br_fdb_test_addr_hook)(struct net_device *dev,
4294 unsigned char *addr) __read_mostly;
4295EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4296#endif
4297
4298static inline struct sk_buff *
4299sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4300 struct net_device *orig_dev)
4301{
4302#ifdef CONFIG_NET_CLS_ACT
4303 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
4304 struct tcf_result cl_res;
4305
4306
4307
4308
4309
4310
4311 if (!miniq)
4312 return skb;
4313
4314 if (*pt_prev) {
4315 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4316 *pt_prev = NULL;
4317 }
4318
4319 qdisc_skb_cb(skb)->pkt_len = skb->len;
4320 skb->tc_at_ingress = 1;
4321 mini_qdisc_bstats_cpu_update(miniq, skb);
4322
4323 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
4324 case TC_ACT_OK:
4325 case TC_ACT_RECLASSIFY:
4326 skb->tc_index = TC_H_MIN(cl_res.classid);
4327 break;
4328 case TC_ACT_SHOT:
4329 mini_qdisc_qstats_cpu_drop(miniq);
4330 kfree_skb(skb);
4331 return NULL;
4332 case TC_ACT_STOLEN:
4333 case TC_ACT_QUEUED:
4334 case TC_ACT_TRAP:
4335 consume_skb(skb);
4336 return NULL;
4337 case TC_ACT_REDIRECT:
4338
4339
4340
4341
4342 __skb_push(skb, skb->mac_len);
4343 skb_do_redirect(skb);
4344 return NULL;
4345 default:
4346 break;
4347 }
4348#endif
4349 return skb;
4350}
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361bool netdev_is_rx_handler_busy(struct net_device *dev)
4362{
4363 ASSERT_RTNL();
4364 return dev && rtnl_dereference(dev->rx_handler);
4365}
4366EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382int netdev_rx_handler_register(struct net_device *dev,
4383 rx_handler_func_t *rx_handler,
4384 void *rx_handler_data)
4385{
4386 if (netdev_is_rx_handler_busy(dev))
4387 return -EBUSY;
4388
4389 if (dev->priv_flags & IFF_NO_RX_HANDLER)
4390 return -EINVAL;
4391
4392
4393 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4394 rcu_assign_pointer(dev->rx_handler, rx_handler);
4395
4396 return 0;
4397}
4398EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408void netdev_rx_handler_unregister(struct net_device *dev)
4409{
4410
4411 ASSERT_RTNL();
4412 RCU_INIT_POINTER(dev->rx_handler, NULL);
4413
4414
4415
4416
4417 synchronize_net();
4418 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4419}
4420EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4421
4422
4423
4424
4425
4426static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4427{
4428 switch (skb->protocol) {
4429 case htons(ETH_P_ARP):
4430 case htons(ETH_P_IP):
4431 case htons(ETH_P_IPV6):
4432 case htons(ETH_P_8021Q):
4433 case htons(ETH_P_8021AD):
4434 return true;
4435 default:
4436 return false;
4437 }
4438}
4439
4440static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4441 int *ret, struct net_device *orig_dev)
4442{
4443#ifdef CONFIG_NETFILTER_INGRESS
4444 if (nf_hook_ingress_active(skb)) {
4445 int ingress_retval;
4446
4447 if (*pt_prev) {
4448 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4449 *pt_prev = NULL;
4450 }
4451
4452 rcu_read_lock();
4453 ingress_retval = nf_hook_ingress(skb);
4454 rcu_read_unlock();
4455 return ingress_retval;
4456 }
4457#endif
4458 return 0;
4459}
4460
4461static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4462{
4463 struct packet_type *ptype, *pt_prev;
4464 rx_handler_func_t *rx_handler;
4465 struct net_device *orig_dev;
4466 bool deliver_exact = false;
4467 int ret = NET_RX_DROP;
4468 __be16 type;
4469
4470 net_timestamp_check(!netdev_tstamp_prequeue, skb);
4471
4472 trace_netif_receive_skb(skb);
4473
4474 orig_dev = skb->dev;
4475
4476 skb_reset_network_header(skb);
4477 if (!skb_transport_header_was_set(skb))
4478 skb_reset_transport_header(skb);
4479 skb_reset_mac_len(skb);
4480
4481 pt_prev = NULL;
4482
4483another_round:
4484 skb->skb_iif = skb->dev->ifindex;
4485
4486 __this_cpu_inc(softnet_data.processed);
4487
4488 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4489 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4490 skb = skb_vlan_untag(skb);
4491 if (unlikely(!skb))
4492 goto out;
4493 }
4494
4495 if (skb_skip_tc_classify(skb))
4496 goto skip_classify;
4497
4498 if (pfmemalloc)
4499 goto skip_taps;
4500
4501 list_for_each_entry_rcu(ptype, &ptype_all, list) {
4502 if (pt_prev)
4503 ret = deliver_skb(skb, pt_prev, orig_dev);
4504 pt_prev = ptype;
4505 }
4506
4507 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4508 if (pt_prev)
4509 ret = deliver_skb(skb, pt_prev, orig_dev);
4510 pt_prev = ptype;
4511 }
4512
4513skip_taps:
4514#ifdef CONFIG_NET_INGRESS
4515 if (static_key_false(&ingress_needed)) {
4516 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4517 if (!skb)
4518 goto out;
4519
4520 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4521 goto out;
4522 }
4523#endif
4524 skb_reset_tc(skb);
4525skip_classify:
4526 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4527 goto drop;
4528
4529 if (skb_vlan_tag_present(skb)) {
4530 if (pt_prev) {
4531 ret = deliver_skb(skb, pt_prev, orig_dev);
4532 pt_prev = NULL;
4533 }
4534 if (vlan_do_receive(&skb))
4535 goto another_round;
4536 else if (unlikely(!skb))
4537 goto out;
4538 }
4539
4540 rx_handler = rcu_dereference(skb->dev->rx_handler);
4541 if (rx_handler) {
4542 if (pt_prev) {
4543 ret = deliver_skb(skb, pt_prev, orig_dev);
4544 pt_prev = NULL;
4545 }
4546 switch (rx_handler(&skb)) {
4547 case RX_HANDLER_CONSUMED:
4548 ret = NET_RX_SUCCESS;
4549 goto out;
4550 case RX_HANDLER_ANOTHER:
4551 goto another_round;
4552 case RX_HANDLER_EXACT:
4553 deliver_exact = true;
4554 case RX_HANDLER_PASS:
4555 break;
4556 default:
4557 BUG();
4558 }
4559 }
4560
4561 if (unlikely(skb_vlan_tag_present(skb))) {
4562 if (skb_vlan_tag_get_id(skb))
4563 skb->pkt_type = PACKET_OTHERHOST;
4564
4565
4566
4567
4568 skb->vlan_tci = 0;
4569 }
4570
4571 type = skb->protocol;
4572
4573
4574 if (likely(!deliver_exact)) {
4575 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4576 &ptype_base[ntohs(type) &
4577 PTYPE_HASH_MASK]);
4578 }
4579
4580 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4581 &orig_dev->ptype_specific);
4582
4583 if (unlikely(skb->dev != orig_dev)) {
4584 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4585 &skb->dev->ptype_specific);
4586 }
4587
4588 if (pt_prev) {
4589 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
4590 goto drop;
4591 else
4592 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4593 } else {
4594drop:
4595 if (!deliver_exact)
4596 atomic_long_inc(&skb->dev->rx_dropped);
4597 else
4598 atomic_long_inc(&skb->dev->rx_nohandler);
4599 kfree_skb(skb);
4600
4601
4602
4603 ret = NET_RX_DROP;
4604 }
4605
4606out:
4607 return ret;
4608}
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625int netif_receive_skb_core(struct sk_buff *skb)
4626{
4627 int ret;
4628
4629 rcu_read_lock();
4630 ret = __netif_receive_skb_core(skb, false);
4631 rcu_read_unlock();
4632
4633 return ret;
4634}
4635EXPORT_SYMBOL(netif_receive_skb_core);
4636
4637static int __netif_receive_skb(struct sk_buff *skb)
4638{
4639 int ret;
4640
4641 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4642 unsigned int noreclaim_flag;
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653 noreclaim_flag = memalloc_noreclaim_save();
4654 ret = __netif_receive_skb_core(skb, true);
4655 memalloc_noreclaim_restore(noreclaim_flag);
4656 } else
4657 ret = __netif_receive_skb_core(skb, false);
4658
4659 return ret;
4660}
4661
4662static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
4663{
4664 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
4665 struct bpf_prog *new = xdp->prog;
4666 int ret = 0;
4667
4668 switch (xdp->command) {
4669 case XDP_SETUP_PROG:
4670 rcu_assign_pointer(dev->xdp_prog, new);
4671 if (old)
4672 bpf_prog_put(old);
4673
4674 if (old && !new) {
4675 static_key_slow_dec(&generic_xdp_needed);
4676 } else if (new && !old) {
4677 static_key_slow_inc(&generic_xdp_needed);
4678 dev_disable_lro(dev);
4679 dev_disable_gro_hw(dev);
4680 }
4681 break;
4682
4683 case XDP_QUERY_PROG:
4684 xdp->prog_attached = !!old;
4685 xdp->prog_id = old ? old->aux->id : 0;
4686 break;
4687
4688 default:
4689 ret = -EINVAL;
4690 break;
4691 }
4692
4693 return ret;
4694}
4695
4696static int netif_receive_skb_internal(struct sk_buff *skb)
4697{
4698 int ret;
4699
4700 net_timestamp_check(netdev_tstamp_prequeue, skb);
4701
4702 if (skb_defer_rx_timestamp(skb))
4703 return NET_RX_SUCCESS;
4704
4705 if (static_key_false(&generic_xdp_needed)) {
4706 int ret;
4707
4708 preempt_disable();
4709 rcu_read_lock();
4710 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4711 rcu_read_unlock();
4712 preempt_enable();
4713
4714 if (ret != XDP_PASS)
4715 return NET_RX_DROP;
4716 }
4717
4718 rcu_read_lock();
4719#ifdef CONFIG_RPS
4720 if (static_key_false(&rps_needed)) {
4721 struct rps_dev_flow voidflow, *rflow = &voidflow;
4722 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4723
4724 if (cpu >= 0) {
4725 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4726 rcu_read_unlock();
4727 return ret;
4728 }
4729 }
4730#endif
4731 ret = __netif_receive_skb(skb);
4732 rcu_read_unlock();
4733 return ret;
4734}
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751int netif_receive_skb(struct sk_buff *skb)
4752{
4753 trace_netif_receive_skb_entry(skb);
4754
4755 return netif_receive_skb_internal(skb);
4756}
4757EXPORT_SYMBOL(netif_receive_skb);
4758
4759DEFINE_PER_CPU(struct work_struct, flush_works);
4760
4761
4762static void flush_backlog(struct work_struct *work)
4763{
4764 struct sk_buff *skb, *tmp;
4765 struct softnet_data *sd;
4766
4767 local_bh_disable();
4768 sd = this_cpu_ptr(&softnet_data);
4769
4770 local_irq_disable();
4771 rps_lock(sd);
4772 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4773 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4774 __skb_unlink(skb, &sd->input_pkt_queue);
4775 kfree_skb(skb);
4776 input_queue_head_incr(sd);
4777 }
4778 }
4779 rps_unlock(sd);
4780 local_irq_enable();
4781
4782 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4783 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4784 __skb_unlink(skb, &sd->process_queue);
4785 kfree_skb(skb);
4786 input_queue_head_incr(sd);
4787 }
4788 }
4789 local_bh_enable();
4790}
4791
4792static void flush_all_backlogs(void)
4793{
4794 unsigned int cpu;
4795
4796 get_online_cpus();
4797
4798 for_each_online_cpu(cpu)
4799 queue_work_on(cpu, system_highpri_wq,
4800 per_cpu_ptr(&flush_works, cpu));
4801
4802 for_each_online_cpu(cpu)
4803 flush_work(per_cpu_ptr(&flush_works, cpu));
4804
4805 put_online_cpus();
4806}
4807
4808static int napi_gro_complete(struct sk_buff *skb)
4809{
4810 struct packet_offload *ptype;
4811 __be16 type = skb->protocol;
4812 struct list_head *head = &offload_base;
4813 int err = -ENOENT;
4814
4815 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4816
4817 if (NAPI_GRO_CB(skb)->count == 1) {
4818 skb_shinfo(skb)->gso_size = 0;
4819 goto out;
4820 }
4821
4822 rcu_read_lock();
4823 list_for_each_entry_rcu(ptype, head, list) {
4824 if (ptype->type != type || !ptype->callbacks.gro_complete)
4825 continue;
4826
4827 err = ptype->callbacks.gro_complete(skb, 0);
4828 break;
4829 }
4830 rcu_read_unlock();
4831
4832 if (err) {
4833 WARN_ON(&ptype->list == head);
4834 kfree_skb(skb);
4835 return NET_RX_SUCCESS;
4836 }
4837
4838out:
4839 return netif_receive_skb_internal(skb);
4840}
4841
4842
4843
4844
4845
4846void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4847{
4848 struct sk_buff *skb, *prev = NULL;
4849
4850
4851 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4852 skb->prev = prev;
4853 prev = skb;
4854 }
4855
4856 for (skb = prev; skb; skb = prev) {
4857 skb->next = NULL;
4858
4859 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4860 return;
4861
4862 prev = skb->prev;
4863 napi_gro_complete(skb);
4864 napi->gro_count--;
4865 }
4866
4867 napi->gro_list = NULL;
4868}
4869EXPORT_SYMBOL(napi_gro_flush);
4870
4871static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4872{
4873 struct sk_buff *p;
4874 unsigned int maclen = skb->dev->hard_header_len;
4875 u32 hash = skb_get_hash_raw(skb);
4876
4877 for (p = napi->gro_list; p; p = p->next) {
4878 unsigned long diffs;
4879
4880 NAPI_GRO_CB(p)->flush = 0;
4881
4882 if (hash != skb_get_hash_raw(p)) {
4883 NAPI_GRO_CB(p)->same_flow = 0;
4884 continue;
4885 }
4886
4887 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4888 diffs |= p->vlan_tci ^ skb->vlan_tci;
4889 diffs |= skb_metadata_dst_cmp(p, skb);
4890 diffs |= skb_metadata_differs(p, skb);
4891 if (maclen == ETH_HLEN)
4892 diffs |= compare_ether_header(skb_mac_header(p),
4893 skb_mac_header(skb));
4894 else if (!diffs)
4895 diffs = memcmp(skb_mac_header(p),
4896 skb_mac_header(skb),
4897 maclen);
4898 NAPI_GRO_CB(p)->same_flow = !diffs;
4899 }
4900}
4901
4902static void skb_gro_reset_offset(struct sk_buff *skb)
4903{
4904 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4905 const skb_frag_t *frag0 = &pinfo->frags[0];
4906
4907 NAPI_GRO_CB(skb)->data_offset = 0;
4908 NAPI_GRO_CB(skb)->frag0 = NULL;
4909 NAPI_GRO_CB(skb)->frag0_len = 0;
4910
4911 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4912 pinfo->nr_frags &&
4913 !PageHighMem(skb_frag_page(frag0))) {
4914 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4915 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4916 skb_frag_size(frag0),
4917 skb->end - skb->tail);
4918 }
4919}
4920
4921static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4922{
4923 struct skb_shared_info *pinfo = skb_shinfo(skb);
4924
4925 BUG_ON(skb->end - skb->tail < grow);
4926
4927 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4928
4929 skb->data_len -= grow;
4930 skb->tail += grow;
4931
4932 pinfo->frags[0].page_offset += grow;
4933 skb_frag_size_sub(&pinfo->frags[0], grow);
4934
4935 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4936 skb_frag_unref(skb, 0);
4937 memmove(pinfo->frags, pinfo->frags + 1,
4938 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4939 }
4940}
4941
4942static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4943{
4944 struct sk_buff **pp = NULL;
4945 struct packet_offload *ptype;
4946 __be16 type = skb->protocol;
4947 struct list_head *head = &offload_base;
4948 int same_flow;
4949 enum gro_result ret;
4950 int grow;
4951
4952 if (netif_elide_gro(skb->dev))
4953 goto normal;
4954
4955 gro_list_prepare(napi, skb);
4956
4957 rcu_read_lock();
4958 list_for_each_entry_rcu(ptype, head, list) {
4959 if (ptype->type != type || !ptype->callbacks.gro_receive)
4960 continue;
4961
4962 skb_set_network_header(skb, skb_gro_offset(skb));
4963 skb_reset_mac_len(skb);
4964 NAPI_GRO_CB(skb)->same_flow = 0;
4965 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4966 NAPI_GRO_CB(skb)->free = 0;
4967 NAPI_GRO_CB(skb)->encap_mark = 0;
4968 NAPI_GRO_CB(skb)->recursion_counter = 0;
4969 NAPI_GRO_CB(skb)->is_fou = 0;
4970 NAPI_GRO_CB(skb)->is_atomic = 1;
4971 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4972
4973
4974 switch (skb->ip_summed) {
4975 case CHECKSUM_COMPLETE:
4976 NAPI_GRO_CB(skb)->csum = skb->csum;
4977 NAPI_GRO_CB(skb)->csum_valid = 1;
4978 NAPI_GRO_CB(skb)->csum_cnt = 0;
4979 break;
4980 case CHECKSUM_UNNECESSARY:
4981 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4982 NAPI_GRO_CB(skb)->csum_valid = 0;
4983 break;
4984 default:
4985 NAPI_GRO_CB(skb)->csum_cnt = 0;
4986 NAPI_GRO_CB(skb)->csum_valid = 0;
4987 }
4988
4989 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4990 break;
4991 }
4992 rcu_read_unlock();
4993
4994 if (&ptype->list == head)
4995 goto normal;
4996
4997 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4998 ret = GRO_CONSUMED;
4999 goto ok;
5000 }
5001
5002 same_flow = NAPI_GRO_CB(skb)->same_flow;
5003 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
5004
5005 if (pp) {
5006 struct sk_buff *nskb = *pp;
5007
5008 *pp = nskb->next;
5009 nskb->next = NULL;
5010 napi_gro_complete(nskb);
5011 napi->gro_count--;
5012 }
5013
5014 if (same_flow)
5015 goto ok;
5016
5017 if (NAPI_GRO_CB(skb)->flush)
5018 goto normal;
5019
5020 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
5021 struct sk_buff *nskb = napi->gro_list;
5022
5023
5024 while (nskb->next) {
5025 pp = &nskb->next;
5026 nskb = *pp;
5027 }
5028 *pp = NULL;
5029 nskb->next = NULL;
5030 napi_gro_complete(nskb);
5031 } else {
5032 napi->gro_count++;
5033 }
5034 NAPI_GRO_CB(skb)->count = 1;
5035 NAPI_GRO_CB(skb)->age = jiffies;
5036 NAPI_GRO_CB(skb)->last = skb;
5037 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
5038 skb->next = napi->gro_list;
5039 napi->gro_list = skb;
5040 ret = GRO_HELD;
5041
5042pull:
5043 grow = skb_gro_offset(skb) - skb_headlen(skb);
5044 if (grow > 0)
5045 gro_pull_from_frag0(skb, grow);
5046ok:
5047 return ret;
5048
5049normal:
5050 ret = GRO_NORMAL;
5051 goto pull;
5052}
5053
5054struct packet_offload *gro_find_receive_by_type(__be16 type)
5055{
5056 struct list_head *offload_head = &offload_base;
5057 struct packet_offload *ptype;
5058
5059 list_for_each_entry_rcu(ptype, offload_head, list) {
5060 if (ptype->type != type || !ptype->callbacks.gro_receive)
5061 continue;
5062 return ptype;
5063 }
5064 return NULL;
5065}
5066EXPORT_SYMBOL(gro_find_receive_by_type);
5067
5068struct packet_offload *gro_find_complete_by_type(__be16 type)
5069{
5070 struct list_head *offload_head = &offload_base;
5071 struct packet_offload *ptype;
5072
5073 list_for_each_entry_rcu(ptype, offload_head, list) {
5074 if (ptype->type != type || !ptype->callbacks.gro_complete)
5075 continue;
5076 return ptype;
5077 }
5078 return NULL;
5079}
5080EXPORT_SYMBOL(gro_find_complete_by_type);
5081
5082static void napi_skb_free_stolen_head(struct sk_buff *skb)
5083{
5084 skb_dst_drop(skb);
5085 secpath_reset(skb);
5086 kmem_cache_free(skbuff_head_cache, skb);
5087}
5088
5089static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
5090{
5091 switch (ret) {
5092 case GRO_NORMAL:
5093 if (netif_receive_skb_internal(skb))
5094 ret = GRO_DROP;
5095 break;
5096
5097 case GRO_DROP:
5098 kfree_skb(skb);
5099 break;
5100
5101 case GRO_MERGED_FREE:
5102 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5103 napi_skb_free_stolen_head(skb);
5104 else
5105 __kfree_skb(skb);
5106 break;
5107
5108 case GRO_HELD:
5109 case GRO_MERGED:
5110 case GRO_CONSUMED:
5111 break;
5112 }
5113
5114 return ret;
5115}
5116
5117gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5118{
5119 skb_mark_napi_id(skb, napi);
5120 trace_napi_gro_receive_entry(skb);
5121
5122 skb_gro_reset_offset(skb);
5123
5124 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
5125}
5126EXPORT_SYMBOL(napi_gro_receive);
5127
5128static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
5129{
5130 if (unlikely(skb->pfmemalloc)) {
5131 consume_skb(skb);
5132 return;
5133 }
5134 __skb_pull(skb, skb_headlen(skb));
5135
5136 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
5137 skb->vlan_tci = 0;
5138 skb->dev = napi->dev;
5139 skb->skb_iif = 0;
5140 skb->encapsulation = 0;
5141 skb_shinfo(skb)->gso_type = 0;
5142 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5143 secpath_reset(skb);
5144
5145 napi->skb = skb;
5146}
5147
5148struct sk_buff *napi_get_frags(struct napi_struct *napi)
5149{
5150 struct sk_buff *skb = napi->skb;
5151
5152 if (!skb) {
5153 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
5154 if (skb) {
5155 napi->skb = skb;
5156 skb_mark_napi_id(skb, napi);
5157 }
5158 }
5159 return skb;
5160}
5161EXPORT_SYMBOL(napi_get_frags);
5162
5163static gro_result_t napi_frags_finish(struct napi_struct *napi,
5164 struct sk_buff *skb,
5165 gro_result_t ret)
5166{
5167 switch (ret) {
5168 case GRO_NORMAL:
5169 case GRO_HELD:
5170 __skb_push(skb, ETH_HLEN);
5171 skb->protocol = eth_type_trans(skb, skb->dev);
5172 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5173 ret = GRO_DROP;
5174 break;
5175
5176 case GRO_DROP:
5177 napi_reuse_skb(napi, skb);
5178 break;
5179
5180 case GRO_MERGED_FREE:
5181 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5182 napi_skb_free_stolen_head(skb);
5183 else
5184 napi_reuse_skb(napi, skb);
5185 break;
5186
5187 case GRO_MERGED:
5188 case GRO_CONSUMED:
5189 break;
5190 }
5191
5192 return ret;
5193}
5194
5195
5196
5197
5198
5199static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
5200{
5201 struct sk_buff *skb = napi->skb;
5202 const struct ethhdr *eth;
5203 unsigned int hlen = sizeof(*eth);
5204
5205 napi->skb = NULL;
5206
5207 skb_reset_mac_header(skb);
5208 skb_gro_reset_offset(skb);
5209
5210 eth = skb_gro_header_fast(skb, 0);
5211 if (unlikely(skb_gro_header_hard(skb, hlen))) {
5212 eth = skb_gro_header_slow(skb, hlen, 0);
5213 if (unlikely(!eth)) {
5214 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5215 __func__, napi->dev->name);
5216 napi_reuse_skb(napi, skb);
5217 return NULL;
5218 }
5219 } else {
5220 gro_pull_from_frag0(skb, hlen);
5221 NAPI_GRO_CB(skb)->frag0 += hlen;
5222 NAPI_GRO_CB(skb)->frag0_len -= hlen;
5223 }
5224 __skb_pull(skb, hlen);
5225
5226
5227
5228
5229
5230
5231 skb->protocol = eth->h_proto;
5232
5233 return skb;
5234}
5235
5236gro_result_t napi_gro_frags(struct napi_struct *napi)
5237{
5238 struct sk_buff *skb = napi_frags_skb(napi);
5239
5240 if (!skb)
5241 return GRO_DROP;
5242
5243 trace_napi_gro_frags_entry(skb);
5244
5245 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5246}
5247EXPORT_SYMBOL(napi_gro_frags);
5248
5249
5250
5251
5252__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5253{
5254 __wsum wsum;
5255 __sum16 sum;
5256
5257 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5258
5259
5260 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5261 if (likely(!sum)) {
5262 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5263 !skb->csum_complete_sw)
5264 netdev_rx_csum_fault(skb->dev);
5265 }
5266
5267 NAPI_GRO_CB(skb)->csum = wsum;
5268 NAPI_GRO_CB(skb)->csum_valid = 1;
5269
5270 return sum;
5271}
5272EXPORT_SYMBOL(__skb_gro_checksum_complete);
5273
5274static void net_rps_send_ipi(struct softnet_data *remsd)
5275{
5276#ifdef CONFIG_RPS
5277 while (remsd) {
5278 struct softnet_data *next = remsd->rps_ipi_next;
5279
5280 if (cpu_online(remsd->cpu))
5281 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5282 remsd = next;
5283 }
5284#endif
5285}
5286
5287
5288
5289
5290
5291static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5292{
5293#ifdef CONFIG_RPS
5294 struct softnet_data *remsd = sd->rps_ipi_list;
5295
5296 if (remsd) {
5297 sd->rps_ipi_list = NULL;
5298
5299 local_irq_enable();
5300
5301
5302 net_rps_send_ipi(remsd);
5303 } else
5304#endif
5305 local_irq_enable();
5306}
5307
5308static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5309{
5310#ifdef CONFIG_RPS
5311 return sd->rps_ipi_list != NULL;
5312#else
5313 return false;
5314#endif
5315}
5316
5317static int process_backlog(struct napi_struct *napi, int quota)
5318{
5319 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5320 bool again = true;
5321 int work = 0;
5322
5323
5324
5325
5326 if (sd_has_rps_ipi_waiting(sd)) {
5327 local_irq_disable();
5328 net_rps_action_and_irq_enable(sd);
5329 }
5330
5331 napi->weight = dev_rx_weight;
5332 while (again) {
5333 struct sk_buff *skb;
5334
5335 while ((skb = __skb_dequeue(&sd->process_queue))) {
5336 rcu_read_lock();
5337 __netif_receive_skb(skb);
5338 rcu_read_unlock();
5339 input_queue_head_incr(sd);
5340 if (++work >= quota)
5341 return work;
5342
5343 }
5344
5345 local_irq_disable();
5346 rps_lock(sd);
5347 if (skb_queue_empty(&sd->input_pkt_queue)) {
5348
5349
5350
5351
5352
5353
5354
5355
5356 napi->state = 0;
5357 again = false;
5358 } else {
5359 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5360 &sd->process_queue);
5361 }
5362 rps_unlock(sd);
5363 local_irq_enable();
5364 }
5365
5366 return work;
5367}
5368
5369
5370
5371
5372
5373
5374
5375
5376void __napi_schedule(struct napi_struct *n)
5377{
5378 unsigned long flags;
5379
5380 local_irq_save(flags);
5381 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5382 local_irq_restore(flags);
5383}
5384EXPORT_SYMBOL(__napi_schedule);
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395bool napi_schedule_prep(struct napi_struct *n)
5396{
5397 unsigned long val, new;
5398
5399 do {
5400 val = READ_ONCE(n->state);
5401 if (unlikely(val & NAPIF_STATE_DISABLE))
5402 return false;
5403 new = val | NAPIF_STATE_SCHED;
5404
5405
5406
5407
5408
5409
5410
5411 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5412 NAPIF_STATE_MISSED;
5413 } while (cmpxchg(&n->state, val, new) != val);
5414
5415 return !(val & NAPIF_STATE_SCHED);
5416}
5417EXPORT_SYMBOL(napi_schedule_prep);
5418
5419
5420
5421
5422
5423
5424
5425void __napi_schedule_irqoff(struct napi_struct *n)
5426{
5427 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5428}
5429EXPORT_SYMBOL(__napi_schedule_irqoff);
5430
5431bool napi_complete_done(struct napi_struct *n, int work_done)
5432{
5433 unsigned long flags, val, new;
5434
5435
5436
5437
5438
5439
5440
5441 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5442 NAPIF_STATE_IN_BUSY_POLL)))
5443 return false;
5444
5445 if (n->gro_list) {
5446 unsigned long timeout = 0;
5447
5448 if (work_done)
5449 timeout = n->dev->gro_flush_timeout;
5450
5451 if (timeout)
5452 hrtimer_start(&n->timer, ns_to_ktime(timeout),
5453 HRTIMER_MODE_REL_PINNED);
5454 else
5455 napi_gro_flush(n, false);
5456 }
5457 if (unlikely(!list_empty(&n->poll_list))) {
5458
5459 local_irq_save(flags);
5460 list_del_init(&n->poll_list);
5461 local_irq_restore(flags);
5462 }
5463
5464 do {
5465 val = READ_ONCE(n->state);
5466
5467 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5468
5469 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5470
5471
5472
5473
5474
5475 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5476 NAPIF_STATE_SCHED;
5477 } while (cmpxchg(&n->state, val, new) != val);
5478
5479 if (unlikely(val & NAPIF_STATE_MISSED)) {
5480 __napi_schedule(n);
5481 return false;
5482 }
5483
5484 return true;
5485}
5486EXPORT_SYMBOL(napi_complete_done);
5487
5488
5489static struct napi_struct *napi_by_id(unsigned int napi_id)
5490{
5491 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5492 struct napi_struct *napi;
5493
5494 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5495 if (napi->napi_id == napi_id)
5496 return napi;
5497
5498 return NULL;
5499}
5500
5501#if defined(CONFIG_NET_RX_BUSY_POLL)
5502
5503#define BUSY_POLL_BUDGET 8
5504
5505static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5506{
5507 int rc;
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518 clear_bit(NAPI_STATE_MISSED, &napi->state);
5519 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5520
5521 local_bh_disable();
5522
5523
5524
5525
5526 rc = napi->poll(napi, BUSY_POLL_BUDGET);
5527 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5528 netpoll_poll_unlock(have_poll_lock);
5529 if (rc == BUSY_POLL_BUDGET)
5530 __napi_schedule(napi);
5531 local_bh_enable();
5532}
5533
5534void napi_busy_loop(unsigned int napi_id,
5535 bool (*loop_end)(void *, unsigned long),
5536 void *loop_end_arg)
5537{
5538 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
5539 int (*napi_poll)(struct napi_struct *napi, int budget);
5540 void *have_poll_lock = NULL;
5541 struct napi_struct *napi;
5542
5543restart:
5544 napi_poll = NULL;
5545
5546 rcu_read_lock();
5547
5548 napi = napi_by_id(napi_id);
5549 if (!napi)
5550 goto out;
5551
5552 preempt_disable();
5553 for (;;) {
5554 int work = 0;
5555
5556 local_bh_disable();
5557 if (!napi_poll) {
5558 unsigned long val = READ_ONCE(napi->state);
5559
5560
5561
5562
5563 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5564 NAPIF_STATE_IN_BUSY_POLL))
5565 goto count;
5566 if (cmpxchg(&napi->state, val,
5567 val | NAPIF_STATE_IN_BUSY_POLL |
5568 NAPIF_STATE_SCHED) != val)
5569 goto count;
5570 have_poll_lock = netpoll_poll_lock(napi);
5571 napi_poll = napi->poll;
5572 }
5573 work = napi_poll(napi, BUSY_POLL_BUDGET);
5574 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
5575count:
5576 if (work > 0)
5577 __NET_ADD_STATS(dev_net(napi->dev),
5578 LINUX_MIB_BUSYPOLLRXPACKETS, work);
5579 local_bh_enable();
5580
5581 if (!loop_end || loop_end(loop_end_arg, start_time))
5582 break;
5583
5584 if (unlikely(need_resched())) {
5585 if (napi_poll)
5586 busy_poll_stop(napi, have_poll_lock);
5587 preempt_enable();
5588 rcu_read_unlock();
5589 cond_resched();
5590 if (loop_end(loop_end_arg, start_time))
5591 return;
5592 goto restart;
5593 }
5594 cpu_relax();
5595 }
5596 if (napi_poll)
5597 busy_poll_stop(napi, have_poll_lock);
5598 preempt_enable();
5599out:
5600 rcu_read_unlock();
5601}
5602EXPORT_SYMBOL(napi_busy_loop);
5603
5604#endif
5605
5606static void napi_hash_add(struct napi_struct *napi)
5607{
5608 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5609 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5610 return;
5611
5612 spin_lock(&napi_hash_lock);
5613
5614
5615 do {
5616 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
5617 napi_gen_id = MIN_NAPI_ID;
5618 } while (napi_by_id(napi_gen_id));
5619 napi->napi_id = napi_gen_id;
5620
5621 hlist_add_head_rcu(&napi->napi_hash_node,
5622 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5623
5624 spin_unlock(&napi_hash_lock);
5625}
5626
5627
5628
5629
5630bool napi_hash_del(struct napi_struct *napi)
5631{
5632 bool rcu_sync_needed = false;
5633
5634 spin_lock(&napi_hash_lock);
5635
5636 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5637 rcu_sync_needed = true;
5638 hlist_del_rcu(&napi->napi_hash_node);
5639 }
5640 spin_unlock(&napi_hash_lock);
5641 return rcu_sync_needed;
5642}
5643EXPORT_SYMBOL_GPL(napi_hash_del);
5644
5645static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5646{
5647 struct napi_struct *napi;
5648
5649 napi = container_of(timer, struct napi_struct, timer);
5650
5651
5652
5653
5654 if (napi->gro_list && !napi_disable_pending(napi) &&
5655 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5656 __napi_schedule_irqoff(napi);
5657
5658 return HRTIMER_NORESTART;
5659}
5660
5661void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5662 int (*poll)(struct napi_struct *, int), int weight)
5663{
5664 INIT_LIST_HEAD(&napi->poll_list);
5665 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5666 napi->timer.function = napi_watchdog;
5667 napi->gro_count = 0;
5668 napi->gro_list = NULL;
5669 napi->skb = NULL;
5670 napi->poll = poll;
5671 if (weight > NAPI_POLL_WEIGHT)
5672 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5673 weight, dev->name);
5674 napi->weight = weight;
5675 list_add(&napi->dev_list, &dev->napi_list);
5676 napi->dev = dev;
5677#ifdef CONFIG_NETPOLL
5678 napi->poll_owner = -1;
5679#endif
5680 set_bit(NAPI_STATE_SCHED, &napi->state);
5681 napi_hash_add(napi);
5682}
5683EXPORT_SYMBOL(netif_napi_add);
5684
5685void napi_disable(struct napi_struct *n)
5686{
5687 might_sleep();
5688 set_bit(NAPI_STATE_DISABLE, &n->state);
5689
5690 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5691 msleep(1);
5692 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5693 msleep(1);
5694
5695 hrtimer_cancel(&n->timer);
5696
5697 clear_bit(NAPI_STATE_DISABLE, &n->state);
5698}
5699EXPORT_SYMBOL(napi_disable);
5700
5701
5702void netif_napi_del(struct napi_struct *napi)
5703{
5704 might_sleep();
5705 if (napi_hash_del(napi))
5706 synchronize_net();
5707 list_del_init(&napi->dev_list);
5708 napi_free_frags(napi);
5709
5710 kfree_skb_list(napi->gro_list);
5711 napi->gro_list = NULL;
5712 napi->gro_count = 0;
5713}
5714EXPORT_SYMBOL(netif_napi_del);
5715
5716static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5717{
5718 void *have;
5719 int work, weight;
5720
5721 list_del_init(&n->poll_list);
5722
5723 have = netpoll_poll_lock(n);
5724
5725 weight = n->weight;
5726
5727
5728
5729
5730
5731
5732
5733 work = 0;
5734 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5735 work = n->poll(n, weight);
5736 trace_napi_poll(n, work, weight);
5737 }
5738
5739 WARN_ON_ONCE(work > weight);
5740
5741 if (likely(work < weight))
5742 goto out_unlock;
5743
5744
5745
5746
5747
5748
5749 if (unlikely(napi_disable_pending(n))) {
5750 napi_complete(n);
5751 goto out_unlock;
5752 }
5753
5754 if (n->gro_list) {
5755
5756
5757
5758 napi_gro_flush(n, HZ >= 1000);
5759 }
5760
5761
5762
5763
5764 if (unlikely(!list_empty(&n->poll_list))) {
5765 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5766 n->dev ? n->dev->name : "backlog");
5767 goto out_unlock;
5768 }
5769
5770 list_add_tail(&n->poll_list, repoll);
5771
5772out_unlock:
5773 netpoll_poll_unlock(have);
5774
5775 return work;
5776}
5777
5778static __latent_entropy void net_rx_action(struct softirq_action *h)
5779{
5780 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5781 unsigned long time_limit = jiffies +
5782 usecs_to_jiffies(netdev_budget_usecs);
5783 int budget = netdev_budget;
5784 LIST_HEAD(list);
5785 LIST_HEAD(repoll);
5786
5787 local_irq_disable();
5788 list_splice_init(&sd->poll_list, &list);
5789 local_irq_enable();
5790
5791 for (;;) {
5792 struct napi_struct *n;
5793
5794 if (list_empty(&list)) {
5795 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5796 goto out;
5797 break;
5798 }
5799
5800 n = list_first_entry(&list, struct napi_struct, poll_list);
5801 budget -= napi_poll(n, &repoll);
5802
5803
5804
5805
5806
5807 if (unlikely(budget <= 0 ||
5808 time_after_eq(jiffies, time_limit))) {
5809 sd->time_squeeze++;
5810 break;
5811 }
5812 }
5813
5814 local_irq_disable();
5815
5816 list_splice_tail_init(&sd->poll_list, &list);
5817 list_splice_tail(&repoll, &list);
5818 list_splice(&list, &sd->poll_list);
5819 if (!list_empty(&sd->poll_list))
5820 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5821
5822 net_rps_action_and_irq_enable(sd);
5823out:
5824 __kfree_skb_flush();
5825}
5826
5827struct netdev_adjacent {
5828 struct net_device *dev;
5829
5830
5831 bool master;
5832
5833
5834 u16 ref_nr;
5835
5836
5837 void *private;
5838
5839 struct list_head list;
5840 struct rcu_head rcu;
5841};
5842
5843static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5844 struct list_head *adj_list)
5845{
5846 struct netdev_adjacent *adj;
5847
5848 list_for_each_entry(adj, adj_list, list) {
5849 if (adj->dev == adj_dev)
5850 return adj;
5851 }
5852 return NULL;
5853}
5854
5855static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5856{
5857 struct net_device *dev = data;
5858
5859 return upper_dev == dev;
5860}
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871bool netdev_has_upper_dev(struct net_device *dev,
5872 struct net_device *upper_dev)
5873{
5874 ASSERT_RTNL();
5875
5876 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5877 upper_dev);
5878}
5879EXPORT_SYMBOL(netdev_has_upper_dev);
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5892 struct net_device *upper_dev)
5893{
5894 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5895 upper_dev);
5896}
5897EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5898
5899
5900
5901
5902
5903
5904
5905
5906bool netdev_has_any_upper_dev(struct net_device *dev)
5907{
5908 ASSERT_RTNL();
5909
5910 return !list_empty(&dev->adj_list.upper);
5911}
5912EXPORT_SYMBOL(netdev_has_any_upper_dev);
5913
5914
5915
5916
5917
5918
5919
5920
5921struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5922{
5923 struct netdev_adjacent *upper;
5924
5925 ASSERT_RTNL();
5926
5927 if (list_empty(&dev->adj_list.upper))
5928 return NULL;
5929
5930 upper = list_first_entry(&dev->adj_list.upper,
5931 struct netdev_adjacent, list);
5932 if (likely(upper->master))
5933 return upper->dev;
5934 return NULL;
5935}
5936EXPORT_SYMBOL(netdev_master_upper_dev_get);
5937
5938
5939
5940
5941
5942
5943
5944
5945static bool netdev_has_any_lower_dev(struct net_device *dev)
5946{
5947 ASSERT_RTNL();
5948
5949 return !list_empty(&dev->adj_list.lower);
5950}
5951
5952void *netdev_adjacent_get_private(struct list_head *adj_list)
5953{
5954 struct netdev_adjacent *adj;
5955
5956 adj = list_entry(adj_list, struct netdev_adjacent, list);
5957
5958 return adj->private;
5959}
5960EXPORT_SYMBOL(netdev_adjacent_get_private);
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5971 struct list_head **iter)
5972{
5973 struct netdev_adjacent *upper;
5974
5975 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5976
5977 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5978
5979 if (&upper->list == &dev->adj_list.upper)
5980 return NULL;
5981
5982 *iter = &upper->list;
5983
5984 return upper->dev;
5985}
5986EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5987
5988static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5989 struct list_head **iter)
5990{
5991 struct netdev_adjacent *upper;
5992
5993 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5994
5995 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5996
5997 if (&upper->list == &dev->adj_list.upper)
5998 return NULL;
5999
6000 *iter = &upper->list;
6001
6002 return upper->dev;
6003}
6004
6005int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6006 int (*fn)(struct net_device *dev,
6007 void *data),
6008 void *data)
6009{
6010 struct net_device *udev;
6011 struct list_head *iter;
6012 int ret;
6013
6014 for (iter = &dev->adj_list.upper,
6015 udev = netdev_next_upper_dev_rcu(dev, &iter);
6016 udev;
6017 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
6018
6019 ret = fn(udev, data);
6020 if (ret)
6021 return ret;
6022
6023
6024 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
6025 if (ret)
6026 return ret;
6027 }
6028
6029 return 0;
6030}
6031EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044void *netdev_lower_get_next_private(struct net_device *dev,
6045 struct list_head **iter)
6046{
6047 struct netdev_adjacent *lower;
6048
6049 lower = list_entry(*iter, struct netdev_adjacent, list);
6050
6051 if (&lower->list == &dev->adj_list.lower)
6052 return NULL;
6053
6054 *iter = lower->list.next;
6055
6056 return lower->private;
6057}
6058EXPORT_SYMBOL(netdev_lower_get_next_private);
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070void *netdev_lower_get_next_private_rcu(struct net_device *dev,
6071 struct list_head **iter)
6072{
6073 struct netdev_adjacent *lower;
6074
6075 WARN_ON_ONCE(!rcu_read_lock_held());
6076
6077 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6078
6079 if (&lower->list == &dev->adj_list.lower)
6080 return NULL;
6081
6082 *iter = &lower->list;
6083
6084 return lower->private;
6085}
6086EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
6100{
6101 struct netdev_adjacent *lower;
6102
6103 lower = list_entry(*iter, struct netdev_adjacent, list);
6104
6105 if (&lower->list == &dev->adj_list.lower)
6106 return NULL;
6107
6108 *iter = lower->list.next;
6109
6110 return lower->dev;
6111}
6112EXPORT_SYMBOL(netdev_lower_get_next);
6113
6114static struct net_device *netdev_next_lower_dev(struct net_device *dev,
6115 struct list_head **iter)
6116{
6117 struct netdev_adjacent *lower;
6118
6119 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
6120
6121 if (&lower->list == &dev->adj_list.lower)
6122 return NULL;
6123
6124 *iter = &lower->list;
6125
6126 return lower->dev;
6127}
6128
6129int netdev_walk_all_lower_dev(struct net_device *dev,
6130 int (*fn)(struct net_device *dev,
6131 void *data),
6132 void *data)
6133{
6134 struct net_device *ldev;
6135 struct list_head *iter;
6136 int ret;
6137
6138 for (iter = &dev->adj_list.lower,
6139 ldev = netdev_next_lower_dev(dev, &iter);
6140 ldev;
6141 ldev = netdev_next_lower_dev(dev, &iter)) {
6142
6143 ret = fn(ldev, data);
6144 if (ret)
6145 return ret;
6146
6147
6148 ret = netdev_walk_all_lower_dev(ldev, fn, data);
6149 if (ret)
6150 return ret;
6151 }
6152
6153 return 0;
6154}
6155EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6156
6157static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
6158 struct list_head **iter)
6159{
6160 struct netdev_adjacent *lower;
6161
6162 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6163 if (&lower->list == &dev->adj_list.lower)
6164 return NULL;
6165
6166 *iter = &lower->list;
6167
6168 return lower->dev;
6169}
6170
6171int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6172 int (*fn)(struct net_device *dev,
6173 void *data),
6174 void *data)
6175{
6176 struct net_device *ldev;
6177 struct list_head *iter;
6178 int ret;
6179
6180 for (iter = &dev->adj_list.lower,
6181 ldev = netdev_next_lower_dev_rcu(dev, &iter);
6182 ldev;
6183 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
6184
6185 ret = fn(ldev, data);
6186 if (ret)
6187 return ret;
6188
6189
6190 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
6191 if (ret)
6192 return ret;
6193 }
6194
6195 return 0;
6196}
6197EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6209{
6210 struct netdev_adjacent *lower;
6211
6212 lower = list_first_or_null_rcu(&dev->adj_list.lower,
6213 struct netdev_adjacent, list);
6214 if (lower)
6215 return lower->private;
6216 return NULL;
6217}
6218EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6219
6220
6221
6222
6223
6224
6225
6226
6227struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6228{
6229 struct netdev_adjacent *upper;
6230
6231 upper = list_first_or_null_rcu(&dev->adj_list.upper,
6232 struct netdev_adjacent, list);
6233 if (upper && likely(upper->master))
6234 return upper->dev;
6235 return NULL;
6236}
6237EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6238
6239static int netdev_adjacent_sysfs_add(struct net_device *dev,
6240 struct net_device *adj_dev,
6241 struct list_head *dev_list)
6242{
6243 char linkname[IFNAMSIZ+7];
6244
6245 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6246 "upper_%s" : "lower_%s", adj_dev->name);
6247 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6248 linkname);
6249}
6250static void netdev_adjacent_sysfs_del(struct net_device *dev,
6251 char *name,
6252 struct list_head *dev_list)
6253{
6254 char linkname[IFNAMSIZ+7];
6255
6256 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6257 "upper_%s" : "lower_%s", name);
6258 sysfs_remove_link(&(dev->dev.kobj), linkname);
6259}
6260
6261static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6262 struct net_device *adj_dev,
6263 struct list_head *dev_list)
6264{
6265 return (dev_list == &dev->adj_list.upper ||
6266 dev_list == &dev->adj_list.lower) &&
6267 net_eq(dev_net(dev), dev_net(adj_dev));
6268}
6269
6270static int __netdev_adjacent_dev_insert(struct net_device *dev,
6271 struct net_device *adj_dev,
6272 struct list_head *dev_list,
6273 void *private, bool master)
6274{
6275 struct netdev_adjacent *adj;
6276 int ret;
6277
6278 adj = __netdev_find_adj(adj_dev, dev_list);
6279
6280 if (adj) {
6281 adj->ref_nr += 1;
6282 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6283 dev->name, adj_dev->name, adj->ref_nr);
6284
6285 return 0;
6286 }
6287
6288 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6289 if (!adj)
6290 return -ENOMEM;
6291
6292 adj->dev = adj_dev;
6293 adj->master = master;
6294 adj->ref_nr = 1;
6295 adj->private = private;
6296 dev_hold(adj_dev);
6297
6298 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6299 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
6300
6301 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
6302 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
6303 if (ret)
6304 goto free_adj;
6305 }
6306
6307
6308 if (master) {
6309 ret = sysfs_create_link(&(dev->dev.kobj),
6310 &(adj_dev->dev.kobj), "master");
6311 if (ret)
6312 goto remove_symlinks;
6313
6314 list_add_rcu(&adj->list, dev_list);
6315 } else {
6316 list_add_tail_rcu(&adj->list, dev_list);
6317 }
6318
6319 return 0;
6320
6321remove_symlinks:
6322 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6323 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6324free_adj:
6325 kfree(adj);
6326 dev_put(adj_dev);
6327
6328 return ret;
6329}
6330
6331static void __netdev_adjacent_dev_remove(struct net_device *dev,
6332 struct net_device *adj_dev,
6333 u16 ref_nr,
6334 struct list_head *dev_list)
6335{
6336 struct netdev_adjacent *adj;
6337
6338 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6339 dev->name, adj_dev->name, ref_nr);
6340
6341 adj = __netdev_find_adj(adj_dev, dev_list);
6342
6343 if (!adj) {
6344 pr_err("Adjacency does not exist for device %s from %s\n",
6345 dev->name, adj_dev->name);
6346 WARN_ON(1);
6347 return;
6348 }
6349
6350 if (adj->ref_nr > ref_nr) {
6351 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6352 dev->name, adj_dev->name, ref_nr,
6353 adj->ref_nr - ref_nr);
6354 adj->ref_nr -= ref_nr;
6355 return;
6356 }
6357
6358 if (adj->master)
6359 sysfs_remove_link(&(dev->dev.kobj), "master");
6360
6361 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6362 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6363
6364 list_del_rcu(&adj->list);
6365 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
6366 adj_dev->name, dev->name, adj_dev->name);
6367 dev_put(adj_dev);
6368 kfree_rcu(adj, rcu);
6369}
6370
6371static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6372 struct net_device *upper_dev,
6373 struct list_head *up_list,
6374 struct list_head *down_list,
6375 void *private, bool master)
6376{
6377 int ret;
6378
6379 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
6380 private, master);
6381 if (ret)
6382 return ret;
6383
6384 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
6385 private, false);
6386 if (ret) {
6387 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
6388 return ret;
6389 }
6390
6391 return 0;
6392}
6393
6394static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6395 struct net_device *upper_dev,
6396 u16 ref_nr,
6397 struct list_head *up_list,
6398 struct list_head *down_list)
6399{
6400 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6401 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
6402}
6403
6404static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6405 struct net_device *upper_dev,
6406 void *private, bool master)
6407{
6408 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6409 &dev->adj_list.upper,
6410 &upper_dev->adj_list.lower,
6411 private, master);
6412}
6413
6414static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6415 struct net_device *upper_dev)
6416{
6417 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
6418 &dev->adj_list.upper,
6419 &upper_dev->adj_list.lower);
6420}
6421
6422static int __netdev_upper_dev_link(struct net_device *dev,
6423 struct net_device *upper_dev, bool master,
6424 void *upper_priv, void *upper_info,
6425 struct netlink_ext_ack *extack)
6426{
6427 struct netdev_notifier_changeupper_info changeupper_info = {
6428 .info = {
6429 .dev = dev,
6430 .extack = extack,
6431 },
6432 .upper_dev = upper_dev,
6433 .master = master,
6434 .linking = true,
6435 .upper_info = upper_info,
6436 };
6437 struct net_device *master_dev;
6438 int ret = 0;
6439
6440 ASSERT_RTNL();
6441
6442 if (dev == upper_dev)
6443 return -EBUSY;
6444
6445
6446 if (netdev_has_upper_dev(upper_dev, dev))
6447 return -EBUSY;
6448
6449 if (!master) {
6450 if (netdev_has_upper_dev(dev, upper_dev))
6451 return -EEXIST;
6452 } else {
6453 master_dev = netdev_master_upper_dev_get(dev);
6454 if (master_dev)
6455 return master_dev == upper_dev ? -EEXIST : -EBUSY;
6456 }
6457
6458 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
6459 &changeupper_info.info);
6460 ret = notifier_to_errno(ret);
6461 if (ret)
6462 return ret;
6463
6464 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
6465 master);
6466 if (ret)
6467 return ret;
6468
6469 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
6470 &changeupper_info.info);
6471 ret = notifier_to_errno(ret);
6472 if (ret)
6473 goto rollback;
6474
6475 return 0;
6476
6477rollback:
6478 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6479
6480 return ret;
6481}
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494int netdev_upper_dev_link(struct net_device *dev,
6495 struct net_device *upper_dev,
6496 struct netlink_ext_ack *extack)
6497{
6498 return __netdev_upper_dev_link(dev, upper_dev, false,
6499 NULL, NULL, extack);
6500}
6501EXPORT_SYMBOL(netdev_upper_dev_link);
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517int netdev_master_upper_dev_link(struct net_device *dev,
6518 struct net_device *upper_dev,
6519 void *upper_priv, void *upper_info,
6520 struct netlink_ext_ack *extack)
6521{
6522 return __netdev_upper_dev_link(dev, upper_dev, true,
6523 upper_priv, upper_info, extack);
6524}
6525EXPORT_SYMBOL(netdev_master_upper_dev_link);
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535void netdev_upper_dev_unlink(struct net_device *dev,
6536 struct net_device *upper_dev)
6537{
6538 struct netdev_notifier_changeupper_info changeupper_info = {
6539 .info = {
6540 .dev = dev,
6541 },
6542 .upper_dev = upper_dev,
6543 .linking = false,
6544 };
6545
6546 ASSERT_RTNL();
6547
6548 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6549
6550 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
6551 &changeupper_info.info);
6552
6553 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6554
6555 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
6556 &changeupper_info.info);
6557}
6558EXPORT_SYMBOL(netdev_upper_dev_unlink);
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568void netdev_bonding_info_change(struct net_device *dev,
6569 struct netdev_bonding_info *bonding_info)
6570{
6571 struct netdev_notifier_bonding_info info = {
6572 .info.dev = dev,
6573 };
6574
6575 memcpy(&info.bonding_info, bonding_info,
6576 sizeof(struct netdev_bonding_info));
6577 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
6578 &info.info);
6579}
6580EXPORT_SYMBOL(netdev_bonding_info_change);
6581
6582static void netdev_adjacent_add_links(struct net_device *dev)
6583{
6584 struct netdev_adjacent *iter;
6585
6586 struct net *net = dev_net(dev);
6587
6588 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6589 if (!net_eq(net, dev_net(iter->dev)))
6590 continue;
6591 netdev_adjacent_sysfs_add(iter->dev, dev,
6592 &iter->dev->adj_list.lower);
6593 netdev_adjacent_sysfs_add(dev, iter->dev,
6594 &dev->adj_list.upper);
6595 }
6596
6597 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6598 if (!net_eq(net, dev_net(iter->dev)))
6599 continue;
6600 netdev_adjacent_sysfs_add(iter->dev, dev,
6601 &iter->dev->adj_list.upper);
6602 netdev_adjacent_sysfs_add(dev, iter->dev,
6603 &dev->adj_list.lower);
6604 }
6605}
6606
6607static void netdev_adjacent_del_links(struct net_device *dev)
6608{
6609 struct netdev_adjacent *iter;
6610
6611 struct net *net = dev_net(dev);
6612
6613 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6614 if (!net_eq(net, dev_net(iter->dev)))
6615 continue;
6616 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6617 &iter->dev->adj_list.lower);
6618 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6619 &dev->adj_list.upper);
6620 }
6621
6622 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6623 if (!net_eq(net, dev_net(iter->dev)))
6624 continue;
6625 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6626 &iter->dev->adj_list.upper);
6627 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6628 &dev->adj_list.lower);
6629 }
6630}
6631
6632void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
6633{
6634 struct netdev_adjacent *iter;
6635
6636 struct net *net = dev_net(dev);
6637
6638 list_for_each_entry(iter, &dev->adj_list.upper, list) {
6639 if (!net_eq(net, dev_net(iter->dev)))
6640 continue;
6641 netdev_adjacent_sysfs_del(iter->dev, oldname,
6642 &iter->dev->adj_list.lower);
6643 netdev_adjacent_sysfs_add(iter->dev, dev,
6644 &iter->dev->adj_list.lower);
6645 }
6646
6647 list_for_each_entry(iter, &dev->adj_list.lower, list) {
6648 if (!net_eq(net, dev_net(iter->dev)))
6649 continue;
6650 netdev_adjacent_sysfs_del(iter->dev, oldname,
6651 &iter->dev->adj_list.upper);
6652 netdev_adjacent_sysfs_add(iter->dev, dev,
6653 &iter->dev->adj_list.upper);
6654 }
6655}
6656
6657void *netdev_lower_dev_get_private(struct net_device *dev,
6658 struct net_device *lower_dev)
6659{
6660 struct netdev_adjacent *lower;
6661
6662 if (!lower_dev)
6663 return NULL;
6664 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
6665 if (!lower)
6666 return NULL;
6667
6668 return lower->private;
6669}
6670EXPORT_SYMBOL(netdev_lower_dev_get_private);
6671
6672
6673int dev_get_nest_level(struct net_device *dev)
6674{
6675 struct net_device *lower = NULL;
6676 struct list_head *iter;
6677 int max_nest = -1;
6678 int nest;
6679
6680 ASSERT_RTNL();
6681
6682 netdev_for_each_lower_dev(dev, lower, iter) {
6683 nest = dev_get_nest_level(lower);
6684 if (max_nest < nest)
6685 max_nest = nest;
6686 }
6687
6688 return max_nest + 1;
6689}
6690EXPORT_SYMBOL(dev_get_nest_level);
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700void netdev_lower_state_changed(struct net_device *lower_dev,
6701 void *lower_state_info)
6702{
6703 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
6704 .info.dev = lower_dev,
6705 };
6706
6707 ASSERT_RTNL();
6708 changelowerstate_info.lower_state_info = lower_state_info;
6709 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
6710 &changelowerstate_info.info);
6711}
6712EXPORT_SYMBOL(netdev_lower_state_changed);
6713
6714static void dev_change_rx_flags(struct net_device *dev, int flags)
6715{
6716 const struct net_device_ops *ops = dev->netdev_ops;
6717
6718 if (ops->ndo_change_rx_flags)
6719 ops->ndo_change_rx_flags(dev, flags);
6720}
6721
6722static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6723{
6724 unsigned int old_flags = dev->flags;
6725 kuid_t uid;
6726 kgid_t gid;
6727
6728 ASSERT_RTNL();
6729
6730 dev->flags |= IFF_PROMISC;
6731 dev->promiscuity += inc;
6732 if (dev->promiscuity == 0) {
6733
6734
6735
6736
6737 if (inc < 0)
6738 dev->flags &= ~IFF_PROMISC;
6739 else {
6740 dev->promiscuity -= inc;
6741 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6742 dev->name);
6743 return -EOVERFLOW;
6744 }
6745 }
6746 if (dev->flags != old_flags) {
6747 pr_info("device %s %s promiscuous mode\n",
6748 dev->name,
6749 dev->flags & IFF_PROMISC ? "entered" : "left");
6750 if (audit_enabled) {
6751 current_uid_gid(&uid, &gid);
6752 audit_log(current->audit_context, GFP_ATOMIC,
6753 AUDIT_ANOM_PROMISCUOUS,
6754 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6755 dev->name, (dev->flags & IFF_PROMISC),
6756 (old_flags & IFF_PROMISC),
6757 from_kuid(&init_user_ns, audit_get_loginuid(current)),
6758 from_kuid(&init_user_ns, uid),
6759 from_kgid(&init_user_ns, gid),
6760 audit_get_sessionid(current));
6761 }
6762
6763 dev_change_rx_flags(dev, IFF_PROMISC);
6764 }
6765 if (notify)
6766 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
6767 return 0;
6768}
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781int dev_set_promiscuity(struct net_device *dev, int inc)
6782{
6783 unsigned int old_flags = dev->flags;
6784 int err;
6785
6786 err = __dev_set_promiscuity(dev, inc, true);
6787 if (err < 0)
6788 return err;
6789 if (dev->flags != old_flags)
6790 dev_set_rx_mode(dev);
6791 return err;
6792}
6793EXPORT_SYMBOL(dev_set_promiscuity);
6794
6795static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6796{
6797 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6798
6799 ASSERT_RTNL();
6800
6801 dev->flags |= IFF_ALLMULTI;
6802 dev->allmulti += inc;
6803 if (dev->allmulti == 0) {
6804
6805
6806
6807
6808 if (inc < 0)
6809 dev->flags &= ~IFF_ALLMULTI;
6810 else {
6811 dev->allmulti -= inc;
6812 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6813 dev->name);
6814 return -EOVERFLOW;
6815 }
6816 }
6817 if (dev->flags ^ old_flags) {
6818 dev_change_rx_flags(dev, IFF_ALLMULTI);
6819 dev_set_rx_mode(dev);
6820 if (notify)
6821 __dev_notify_flags(dev, old_flags,
6822 dev->gflags ^ old_gflags);
6823 }
6824 return 0;
6825}
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840int dev_set_allmulti(struct net_device *dev, int inc)
6841{
6842 return __dev_set_allmulti(dev, inc, true);
6843}
6844EXPORT_SYMBOL(dev_set_allmulti);
6845
6846
6847
6848
6849
6850
6851
6852void __dev_set_rx_mode(struct net_device *dev)
6853{
6854 const struct net_device_ops *ops = dev->netdev_ops;
6855
6856
6857 if (!(dev->flags&IFF_UP))
6858 return;
6859
6860 if (!netif_device_present(dev))
6861 return;
6862
6863 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6864
6865
6866
6867 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6868 __dev_set_promiscuity(dev, 1, false);
6869 dev->uc_promisc = true;
6870 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6871 __dev_set_promiscuity(dev, -1, false);
6872 dev->uc_promisc = false;
6873 }
6874 }
6875
6876 if (ops->ndo_set_rx_mode)
6877 ops->ndo_set_rx_mode(dev);
6878}
6879
6880void dev_set_rx_mode(struct net_device *dev)
6881{
6882 netif_addr_lock_bh(dev);
6883 __dev_set_rx_mode(dev);
6884 netif_addr_unlock_bh(dev);
6885}
6886
6887
6888
6889
6890
6891
6892
6893unsigned int dev_get_flags(const struct net_device *dev)
6894{
6895 unsigned int flags;
6896
6897 flags = (dev->flags & ~(IFF_PROMISC |
6898 IFF_ALLMULTI |
6899 IFF_RUNNING |
6900 IFF_LOWER_UP |
6901 IFF_DORMANT)) |
6902 (dev->gflags & (IFF_PROMISC |
6903 IFF_ALLMULTI));
6904
6905 if (netif_running(dev)) {
6906 if (netif_oper_up(dev))
6907 flags |= IFF_RUNNING;
6908 if (netif_carrier_ok(dev))
6909 flags |= IFF_LOWER_UP;
6910 if (netif_dormant(dev))
6911 flags |= IFF_DORMANT;
6912 }
6913
6914 return flags;
6915}
6916EXPORT_SYMBOL(dev_get_flags);
6917
6918int __dev_change_flags(struct net_device *dev, unsigned int flags)
6919{
6920 unsigned int old_flags = dev->flags;
6921 int ret;
6922
6923 ASSERT_RTNL();
6924
6925
6926
6927
6928
6929 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6930 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6931 IFF_AUTOMEDIA)) |
6932 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6933 IFF_ALLMULTI));
6934
6935
6936
6937
6938
6939 if ((old_flags ^ flags) & IFF_MULTICAST)
6940 dev_change_rx_flags(dev, IFF_MULTICAST);
6941
6942 dev_set_rx_mode(dev);
6943
6944
6945
6946
6947
6948
6949
6950 ret = 0;
6951 if ((old_flags ^ flags) & IFF_UP) {
6952 if (old_flags & IFF_UP)
6953 __dev_close(dev);
6954 else
6955 ret = __dev_open(dev);
6956 }
6957
6958 if ((flags ^ dev->gflags) & IFF_PROMISC) {
6959 int inc = (flags & IFF_PROMISC) ? 1 : -1;
6960 unsigned int old_flags = dev->flags;
6961
6962 dev->gflags ^= IFF_PROMISC;
6963
6964 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6965 if (dev->flags != old_flags)
6966 dev_set_rx_mode(dev);
6967 }
6968
6969
6970
6971
6972
6973 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6974 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6975
6976 dev->gflags ^= IFF_ALLMULTI;
6977 __dev_set_allmulti(dev, inc, false);
6978 }
6979
6980 return ret;
6981}
6982
6983void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6984 unsigned int gchanges)
6985{
6986 unsigned int changes = dev->flags ^ old_flags;
6987
6988 if (gchanges)
6989 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6990
6991 if (changes & IFF_UP) {
6992 if (dev->flags & IFF_UP)
6993 call_netdevice_notifiers(NETDEV_UP, dev);
6994 else
6995 call_netdevice_notifiers(NETDEV_DOWN, dev);
6996 }
6997
6998 if (dev->flags & IFF_UP &&
6999 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
7000 struct netdev_notifier_change_info change_info = {
7001 .info = {
7002 .dev = dev,
7003 },
7004 .flags_changed = changes,
7005 };
7006
7007 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
7008 }
7009}
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019int dev_change_flags(struct net_device *dev, unsigned int flags)
7020{
7021 int ret;
7022 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
7023
7024 ret = __dev_change_flags(dev, flags);
7025 if (ret < 0)
7026 return ret;
7027
7028 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
7029 __dev_notify_flags(dev, old_flags, changes);
7030 return ret;
7031}
7032EXPORT_SYMBOL(dev_change_flags);
7033
7034int __dev_set_mtu(struct net_device *dev, int new_mtu)
7035{
7036 const struct net_device_ops *ops = dev->netdev_ops;
7037
7038 if (ops->ndo_change_mtu)
7039 return ops->ndo_change_mtu(dev, new_mtu);
7040
7041 dev->mtu = new_mtu;
7042 return 0;
7043}
7044EXPORT_SYMBOL(__dev_set_mtu);
7045
7046
7047
7048
7049
7050
7051
7052
7053int dev_set_mtu(struct net_device *dev, int new_mtu)
7054{
7055 int err, orig_mtu;
7056
7057 if (new_mtu == dev->mtu)
7058 return 0;
7059
7060
7061 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
7062 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
7063 dev->name, new_mtu, dev->min_mtu);
7064 return -EINVAL;
7065 }
7066
7067 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
7068 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
7069 dev->name, new_mtu, dev->max_mtu);
7070 return -EINVAL;
7071 }
7072
7073 if (!netif_device_present(dev))
7074 return -ENODEV;
7075
7076 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
7077 err = notifier_to_errno(err);
7078 if (err)
7079 return err;
7080
7081 orig_mtu = dev->mtu;
7082 err = __dev_set_mtu(dev, new_mtu);
7083
7084 if (!err) {
7085 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
7086 err = notifier_to_errno(err);
7087 if (err) {
7088
7089
7090
7091 __dev_set_mtu(dev, orig_mtu);
7092 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
7093 }
7094 }
7095 return err;
7096}
7097EXPORT_SYMBOL(dev_set_mtu);
7098
7099
7100
7101
7102
7103
7104int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
7105{
7106 unsigned int orig_len = dev->tx_queue_len;
7107 int res;
7108
7109 if (new_len != (unsigned int)new_len)
7110 return -ERANGE;
7111
7112 if (new_len != orig_len) {
7113 dev->tx_queue_len = new_len;
7114 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
7115 res = notifier_to_errno(res);
7116 if (res) {
7117 netdev_err(dev,
7118 "refused to change device tx_queue_len\n");
7119 dev->tx_queue_len = orig_len;
7120 return res;
7121 }
7122 return dev_qdisc_change_tx_queue_len(dev);
7123 }
7124
7125 return 0;
7126}
7127
7128
7129
7130
7131
7132
7133void dev_set_group(struct net_device *dev, int new_group)
7134{
7135 dev->group = new_group;
7136}
7137EXPORT_SYMBOL(dev_set_group);
7138
7139
7140
7141
7142
7143
7144
7145
7146int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
7147{
7148 const struct net_device_ops *ops = dev->netdev_ops;
7149 int err;
7150
7151 if (!ops->ndo_set_mac_address)
7152 return -EOPNOTSUPP;
7153 if (sa->sa_family != dev->type)
7154 return -EINVAL;
7155 if (!netif_device_present(dev))
7156 return -ENODEV;
7157 err = ops->ndo_set_mac_address(dev, sa);
7158 if (err)
7159 return err;
7160 dev->addr_assign_type = NET_ADDR_SET;
7161 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
7162 add_device_randomness(dev->dev_addr, dev->addr_len);
7163 return 0;
7164}
7165EXPORT_SYMBOL(dev_set_mac_address);
7166
7167
7168
7169
7170
7171
7172
7173
7174int dev_change_carrier(struct net_device *dev, bool new_carrier)
7175{
7176 const struct net_device_ops *ops = dev->netdev_ops;
7177
7178 if (!ops->ndo_change_carrier)
7179 return -EOPNOTSUPP;
7180 if (!netif_device_present(dev))
7181 return -ENODEV;
7182 return ops->ndo_change_carrier(dev, new_carrier);
7183}
7184EXPORT_SYMBOL(dev_change_carrier);
7185
7186
7187
7188
7189
7190
7191
7192
7193int dev_get_phys_port_id(struct net_device *dev,
7194 struct netdev_phys_item_id *ppid)
7195{
7196 const struct net_device_ops *ops = dev->netdev_ops;
7197
7198 if (!ops->ndo_get_phys_port_id)
7199 return -EOPNOTSUPP;
7200 return ops->ndo_get_phys_port_id(dev, ppid);
7201}
7202EXPORT_SYMBOL(dev_get_phys_port_id);
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212int dev_get_phys_port_name(struct net_device *dev,
7213 char *name, size_t len)
7214{
7215 const struct net_device_ops *ops = dev->netdev_ops;
7216
7217 if (!ops->ndo_get_phys_port_name)
7218 return -EOPNOTSUPP;
7219 return ops->ndo_get_phys_port_name(dev, name, len);
7220}
7221EXPORT_SYMBOL(dev_get_phys_port_name);
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231int dev_change_proto_down(struct net_device *dev, bool proto_down)
7232{
7233 const struct net_device_ops *ops = dev->netdev_ops;
7234
7235 if (!ops->ndo_change_proto_down)
7236 return -EOPNOTSUPP;
7237 if (!netif_device_present(dev))
7238 return -ENODEV;
7239 return ops->ndo_change_proto_down(dev, proto_down);
7240}
7241EXPORT_SYMBOL(dev_change_proto_down);
7242
7243void __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
7244 struct netdev_bpf *xdp)
7245{
7246 memset(xdp, 0, sizeof(*xdp));
7247 xdp->command = XDP_QUERY_PROG;
7248
7249
7250 WARN_ON(bpf_op(dev, xdp) < 0);
7251}
7252
7253static u8 __dev_xdp_attached(struct net_device *dev, bpf_op_t bpf_op)
7254{
7255 struct netdev_bpf xdp;
7256
7257 __dev_xdp_query(dev, bpf_op, &xdp);
7258
7259 return xdp.prog_attached;
7260}
7261
7262static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
7263 struct netlink_ext_ack *extack, u32 flags,
7264 struct bpf_prog *prog)
7265{
7266 struct netdev_bpf xdp;
7267
7268 memset(&xdp, 0, sizeof(xdp));
7269 if (flags & XDP_FLAGS_HW_MODE)
7270 xdp.command = XDP_SETUP_PROG_HW;
7271 else
7272 xdp.command = XDP_SETUP_PROG;
7273 xdp.extack = extack;
7274 xdp.flags = flags;
7275 xdp.prog = prog;
7276
7277 return bpf_op(dev, &xdp);
7278}
7279
7280static void dev_xdp_uninstall(struct net_device *dev)
7281{
7282 struct netdev_bpf xdp;
7283 bpf_op_t ndo_bpf;
7284
7285
7286 WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL));
7287
7288
7289 ndo_bpf = dev->netdev_ops->ndo_bpf;
7290 if (!ndo_bpf)
7291 return;
7292
7293 __dev_xdp_query(dev, ndo_bpf, &xdp);
7294 if (xdp.prog_attached == XDP_ATTACHED_NONE)
7295 return;
7296
7297
7298 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, NULL));
7299}
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
7311 int fd, u32 flags)
7312{
7313 const struct net_device_ops *ops = dev->netdev_ops;
7314 struct bpf_prog *prog = NULL;
7315 bpf_op_t bpf_op, bpf_chk;
7316 int err;
7317
7318 ASSERT_RTNL();
7319
7320 bpf_op = bpf_chk = ops->ndo_bpf;
7321 if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
7322 return -EOPNOTSUPP;
7323 if (!bpf_op || (flags & XDP_FLAGS_SKB_MODE))
7324 bpf_op = generic_xdp_install;
7325 if (bpf_op == bpf_chk)
7326 bpf_chk = generic_xdp_install;
7327
7328 if (fd >= 0) {
7329 if (bpf_chk && __dev_xdp_attached(dev, bpf_chk))
7330 return -EEXIST;
7331 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
7332 __dev_xdp_attached(dev, bpf_op))
7333 return -EBUSY;
7334
7335 prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
7336 bpf_op == ops->ndo_bpf);
7337 if (IS_ERR(prog))
7338 return PTR_ERR(prog);
7339
7340 if (!(flags & XDP_FLAGS_HW_MODE) &&
7341 bpf_prog_is_dev_bound(prog->aux)) {
7342 NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported");
7343 bpf_prog_put(prog);
7344 return -EINVAL;
7345 }
7346 }
7347
7348 err = dev_xdp_install(dev, bpf_op, extack, flags, prog);
7349 if (err < 0 && prog)
7350 bpf_prog_put(prog);
7351
7352 return err;
7353}
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363static int dev_new_index(struct net *net)
7364{
7365 int ifindex = net->ifindex;
7366
7367 for (;;) {
7368 if (++ifindex <= 0)
7369 ifindex = 1;
7370 if (!__dev_get_by_index(net, ifindex))
7371 return net->ifindex = ifindex;
7372 }
7373}
7374
7375
7376static LIST_HEAD(net_todo_list);
7377DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
7378
7379static void net_set_todo(struct net_device *dev)
7380{
7381 list_add_tail(&dev->todo_list, &net_todo_list);
7382 dev_net(dev)->dev_unreg_count++;
7383}
7384
7385static void rollback_registered_many(struct list_head *head)
7386{
7387 struct net_device *dev, *tmp;
7388 LIST_HEAD(close_head);
7389
7390 BUG_ON(dev_boot_phase);
7391 ASSERT_RTNL();
7392
7393 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
7394
7395
7396
7397
7398 if (dev->reg_state == NETREG_UNINITIALIZED) {
7399 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
7400 dev->name, dev);
7401
7402 WARN_ON(1);
7403 list_del(&dev->unreg_list);
7404 continue;
7405 }
7406 dev->dismantle = true;
7407 BUG_ON(dev->reg_state != NETREG_REGISTERED);
7408 }
7409
7410
7411 list_for_each_entry(dev, head, unreg_list)
7412 list_add_tail(&dev->close_list, &close_head);
7413 dev_close_many(&close_head, true);
7414
7415 list_for_each_entry(dev, head, unreg_list) {
7416
7417 unlist_netdevice(dev);
7418
7419 dev->reg_state = NETREG_UNREGISTERING;
7420 }
7421 flush_all_backlogs();
7422
7423 synchronize_net();
7424
7425 list_for_each_entry(dev, head, unreg_list) {
7426 struct sk_buff *skb = NULL;
7427
7428
7429 dev_shutdown(dev);
7430
7431 dev_xdp_uninstall(dev);
7432
7433
7434
7435
7436 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7437
7438 if (!dev->rtnl_link_ops ||
7439 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7440 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
7441 GFP_KERNEL, NULL, 0);
7442
7443
7444
7445
7446 dev_uc_flush(dev);
7447 dev_mc_flush(dev);
7448
7449 if (dev->netdev_ops->ndo_uninit)
7450 dev->netdev_ops->ndo_uninit(dev);
7451
7452 if (skb)
7453 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
7454
7455
7456 WARN_ON(netdev_has_any_upper_dev(dev));
7457 WARN_ON(netdev_has_any_lower_dev(dev));
7458
7459
7460 netdev_unregister_kobject(dev);
7461#ifdef CONFIG_XPS
7462
7463 netif_reset_xps_queues_gt(dev, 0);
7464#endif
7465 }
7466
7467 synchronize_net();
7468
7469 list_for_each_entry(dev, head, unreg_list)
7470 dev_put(dev);
7471}
7472
7473static void rollback_registered(struct net_device *dev)
7474{
7475 LIST_HEAD(single);
7476
7477 list_add(&dev->unreg_list, &single);
7478 rollback_registered_many(&single);
7479 list_del(&single);
7480}
7481
7482static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
7483 struct net_device *upper, netdev_features_t features)
7484{
7485 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7486 netdev_features_t feature;
7487 int feature_bit;
7488
7489 for_each_netdev_feature(&upper_disables, feature_bit) {
7490 feature = __NETIF_F_BIT(feature_bit);
7491 if (!(upper->wanted_features & feature)
7492 && (features & feature)) {
7493 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
7494 &feature, upper->name);
7495 features &= ~feature;
7496 }
7497 }
7498
7499 return features;
7500}
7501
7502static void netdev_sync_lower_features(struct net_device *upper,
7503 struct net_device *lower, netdev_features_t features)
7504{
7505 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
7506 netdev_features_t feature;
7507 int feature_bit;
7508
7509 for_each_netdev_feature(&upper_disables, feature_bit) {
7510 feature = __NETIF_F_BIT(feature_bit);
7511 if (!(features & feature) && (lower->features & feature)) {
7512 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
7513 &feature, lower->name);
7514 lower->wanted_features &= ~feature;
7515 netdev_update_features(lower);
7516
7517 if (unlikely(lower->features & feature))
7518 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
7519 &feature, lower->name);
7520 }
7521 }
7522}
7523
7524static netdev_features_t netdev_fix_features(struct net_device *dev,
7525 netdev_features_t features)
7526{
7527
7528 if ((features & NETIF_F_HW_CSUM) &&
7529 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
7530 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
7531 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
7532 }
7533
7534
7535 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
7536 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
7537 features &= ~NETIF_F_ALL_TSO;
7538 }
7539
7540 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
7541 !(features & NETIF_F_IP_CSUM)) {
7542 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
7543 features &= ~NETIF_F_TSO;
7544 features &= ~NETIF_F_TSO_ECN;
7545 }
7546
7547 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
7548 !(features & NETIF_F_IPV6_CSUM)) {
7549 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
7550 features &= ~NETIF_F_TSO6;
7551 }
7552
7553
7554 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
7555 features &= ~NETIF_F_TSO_MANGLEID;
7556
7557
7558 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
7559 features &= ~NETIF_F_TSO_ECN;
7560
7561
7562 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
7563 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
7564 features &= ~NETIF_F_GSO;
7565 }
7566
7567
7568 if ((features & dev->gso_partial_features) &&
7569 !(features & NETIF_F_GSO_PARTIAL)) {
7570 netdev_dbg(dev,
7571 "Dropping partially supported GSO features since no GSO partial.\n");
7572 features &= ~dev->gso_partial_features;
7573 }
7574
7575 if (!(features & NETIF_F_RXCSUM)) {
7576
7577
7578
7579
7580
7581 if (features & NETIF_F_GRO_HW) {
7582 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
7583 features &= ~NETIF_F_GRO_HW;
7584 }
7585 }
7586
7587
7588 if (features & NETIF_F_RXFCS) {
7589 if (features & NETIF_F_LRO) {
7590 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
7591 features &= ~NETIF_F_LRO;
7592 }
7593
7594 if (features & NETIF_F_GRO_HW) {
7595 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
7596 features &= ~NETIF_F_GRO_HW;
7597 }
7598 }
7599
7600 return features;
7601}
7602
7603int __netdev_update_features(struct net_device *dev)
7604{
7605 struct net_device *upper, *lower;
7606 netdev_features_t features;
7607 struct list_head *iter;
7608 int err = -1;
7609
7610 ASSERT_RTNL();
7611
7612 features = netdev_get_wanted_features(dev);
7613
7614 if (dev->netdev_ops->ndo_fix_features)
7615 features = dev->netdev_ops->ndo_fix_features(dev, features);
7616
7617
7618 features = netdev_fix_features(dev, features);
7619
7620
7621 netdev_for_each_upper_dev_rcu(dev, upper, iter)
7622 features = netdev_sync_upper_features(dev, upper, features);
7623
7624 if (dev->features == features)
7625 goto sync_lower;
7626
7627 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7628 &dev->features, &features);
7629
7630 if (dev->netdev_ops->ndo_set_features)
7631 err = dev->netdev_ops->ndo_set_features(dev, features);
7632 else
7633 err = 0;
7634
7635 if (unlikely(err < 0)) {
7636 netdev_err(dev,
7637 "set_features() failed (%d); wanted %pNF, left %pNF\n",
7638 err, &features, &dev->features);
7639
7640
7641
7642 return -1;
7643 }
7644
7645sync_lower:
7646
7647
7648
7649 netdev_for_each_lower_dev(dev, lower, iter)
7650 netdev_sync_lower_features(dev, lower, features);
7651
7652 if (!err) {
7653 netdev_features_t diff = features ^ dev->features;
7654
7655 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
7656
7657
7658
7659
7660
7661
7662
7663 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
7664 dev->features = features;
7665 udp_tunnel_get_rx_info(dev);
7666 } else {
7667 udp_tunnel_drop_rx_info(dev);
7668 }
7669 }
7670
7671 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
7672 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
7673 dev->features = features;
7674 err |= vlan_get_rx_ctag_filter_info(dev);
7675 } else {
7676 vlan_drop_rx_ctag_filter_info(dev);
7677 }
7678 }
7679
7680 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
7681 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
7682 dev->features = features;
7683 err |= vlan_get_rx_stag_filter_info(dev);
7684 } else {
7685 vlan_drop_rx_stag_filter_info(dev);
7686 }
7687 }
7688
7689 dev->features = features;
7690 }
7691
7692 return err < 0 ? 0 : 1;
7693}
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703void netdev_update_features(struct net_device *dev)
7704{
7705 if (__netdev_update_features(dev))
7706 netdev_features_change(dev);
7707}
7708EXPORT_SYMBOL(netdev_update_features);
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720void netdev_change_features(struct net_device *dev)
7721{
7722 __netdev_update_features(dev);
7723 netdev_features_change(dev);
7724}
7725EXPORT_SYMBOL(netdev_change_features);
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7737 struct net_device *dev)
7738{
7739 if (rootdev->operstate == IF_OPER_DORMANT)
7740 netif_dormant_on(dev);
7741 else
7742 netif_dormant_off(dev);
7743
7744 if (netif_carrier_ok(rootdev))
7745 netif_carrier_on(dev);
7746 else
7747 netif_carrier_off(dev);
7748}
7749EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7750
7751static int netif_alloc_rx_queues(struct net_device *dev)
7752{
7753 unsigned int i, count = dev->num_rx_queues;
7754 struct netdev_rx_queue *rx;
7755 size_t sz = count * sizeof(*rx);
7756 int err = 0;
7757
7758 BUG_ON(count < 1);
7759
7760 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7761 if (!rx)
7762 return -ENOMEM;
7763
7764 dev->_rx = rx;
7765
7766 for (i = 0; i < count; i++) {
7767 rx[i].dev = dev;
7768
7769
7770 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i);
7771 if (err < 0)
7772 goto err_rxq_info;
7773 }
7774 return 0;
7775
7776err_rxq_info:
7777
7778 while (i--)
7779 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
7780 kvfree(dev->_rx);
7781 dev->_rx = NULL;
7782 return err;
7783}
7784
7785static void netif_free_rx_queues(struct net_device *dev)
7786{
7787 unsigned int i, count = dev->num_rx_queues;
7788
7789
7790 if (!dev->_rx)
7791 return;
7792
7793 for (i = 0; i < count; i++)
7794 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
7795
7796 kvfree(dev->_rx);
7797}
7798
7799static void netdev_init_one_queue(struct net_device *dev,
7800 struct netdev_queue *queue, void *_unused)
7801{
7802
7803 spin_lock_init(&queue->_xmit_lock);
7804 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7805 queue->xmit_lock_owner = -1;
7806 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
7807 queue->dev = dev;
7808#ifdef CONFIG_BQL
7809 dql_init(&queue->dql, HZ);
7810#endif
7811}
7812
7813static void netif_free_tx_queues(struct net_device *dev)
7814{
7815 kvfree(dev->_tx);
7816}
7817
7818static int netif_alloc_netdev_queues(struct net_device *dev)
7819{
7820 unsigned int count = dev->num_tx_queues;
7821 struct netdev_queue *tx;
7822 size_t sz = count * sizeof(*tx);
7823
7824 if (count < 1 || count > 0xffff)
7825 return -EINVAL;
7826
7827 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
7828 if (!tx)
7829 return -ENOMEM;
7830
7831 dev->_tx = tx;
7832
7833 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7834 spin_lock_init(&dev->tx_global_lock);
7835
7836 return 0;
7837}
7838
7839void netif_tx_stop_all_queues(struct net_device *dev)
7840{
7841 unsigned int i;
7842
7843 for (i = 0; i < dev->num_tx_queues; i++) {
7844 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7845
7846 netif_tx_stop_queue(txq);
7847 }
7848}
7849EXPORT_SYMBOL(netif_tx_stop_all_queues);
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868int register_netdevice(struct net_device *dev)
7869{
7870 int ret;
7871 struct net *net = dev_net(dev);
7872
7873 BUG_ON(dev_boot_phase);
7874 ASSERT_RTNL();
7875
7876 might_sleep();
7877
7878
7879 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
7880 BUG_ON(!net);
7881
7882 spin_lock_init(&dev->addr_list_lock);
7883 netdev_set_addr_lockdep_class(dev);
7884
7885 ret = dev_get_valid_name(net, dev, dev->name);
7886 if (ret < 0)
7887 goto out;
7888
7889
7890 if (dev->netdev_ops->ndo_init) {
7891 ret = dev->netdev_ops->ndo_init(dev);
7892 if (ret) {
7893 if (ret > 0)
7894 ret = -EIO;
7895 goto out;
7896 }
7897 }
7898
7899 if (((dev->hw_features | dev->features) &
7900 NETIF_F_HW_VLAN_CTAG_FILTER) &&
7901 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7902 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7903 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7904 ret = -EINVAL;
7905 goto err_uninit;
7906 }
7907
7908 ret = -EBUSY;
7909 if (!dev->ifindex)
7910 dev->ifindex = dev_new_index(net);
7911 else if (__dev_get_by_index(net, dev->ifindex))
7912 goto err_uninit;
7913
7914
7915
7916
7917 dev->hw_features |= NETIF_F_SOFT_FEATURES;
7918 dev->features |= NETIF_F_SOFT_FEATURES;
7919
7920 if (dev->netdev_ops->ndo_udp_tunnel_add) {
7921 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7922 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
7923 }
7924
7925 dev->wanted_features = dev->features & dev->hw_features;
7926
7927 if (!(dev->flags & IFF_LOOPBACK))
7928 dev->hw_features |= NETIF_F_NOCACHE_COPY;
7929
7930
7931
7932
7933
7934
7935 if (dev->hw_features & NETIF_F_TSO)
7936 dev->hw_features |= NETIF_F_TSO_MANGLEID;
7937 if (dev->vlan_features & NETIF_F_TSO)
7938 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7939 if (dev->mpls_features & NETIF_F_TSO)
7940 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7941 if (dev->hw_enc_features & NETIF_F_TSO)
7942 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
7943
7944
7945
7946 dev->vlan_features |= NETIF_F_HIGHDMA;
7947
7948
7949
7950 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
7951
7952
7953
7954 dev->mpls_features |= NETIF_F_SG;
7955
7956 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7957 ret = notifier_to_errno(ret);
7958 if (ret)
7959 goto err_uninit;
7960
7961 ret = netdev_register_kobject(dev);
7962 if (ret)
7963 goto err_uninit;
7964 dev->reg_state = NETREG_REGISTERED;
7965
7966 __netdev_update_features(dev);
7967
7968
7969
7970
7971
7972
7973 set_bit(__LINK_STATE_PRESENT, &dev->state);
7974
7975 linkwatch_init_dev(dev);
7976
7977 dev_init_scheduler(dev);
7978 dev_hold(dev);
7979 list_netdevice(dev);
7980 add_device_randomness(dev->dev_addr, dev->addr_len);
7981
7982
7983
7984
7985
7986 if (dev->addr_assign_type == NET_ADDR_PERM)
7987 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7988
7989
7990 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7991 ret = notifier_to_errno(ret);
7992 if (ret) {
7993 rollback_registered(dev);
7994 dev->reg_state = NETREG_UNREGISTERED;
7995 }
7996
7997
7998
7999
8000 if (!dev->rtnl_link_ops ||
8001 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
8002 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
8003
8004out:
8005 return ret;
8006
8007err_uninit:
8008 if (dev->netdev_ops->ndo_uninit)
8009 dev->netdev_ops->ndo_uninit(dev);
8010 if (dev->priv_destructor)
8011 dev->priv_destructor(dev);
8012 goto out;
8013}
8014EXPORT_SYMBOL(register_netdevice);
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026int init_dummy_netdev(struct net_device *dev)
8027{
8028
8029
8030
8031
8032
8033 memset(dev, 0, sizeof(struct net_device));
8034
8035
8036
8037
8038 dev->reg_state = NETREG_DUMMY;
8039
8040
8041 INIT_LIST_HEAD(&dev->napi_list);
8042
8043
8044 set_bit(__LINK_STATE_PRESENT, &dev->state);
8045 set_bit(__LINK_STATE_START, &dev->state);
8046
8047
8048
8049
8050
8051
8052 return 0;
8053}
8054EXPORT_SYMBOL_GPL(init_dummy_netdev);
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070int register_netdev(struct net_device *dev)
8071{
8072 int err;
8073
8074 if (rtnl_lock_killable())
8075 return -EINTR;
8076 err = register_netdevice(dev);
8077 rtnl_unlock();
8078 return err;
8079}
8080EXPORT_SYMBOL(register_netdev);
8081
8082int netdev_refcnt_read(const struct net_device *dev)
8083{
8084 int i, refcnt = 0;
8085
8086 for_each_possible_cpu(i)
8087 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
8088 return refcnt;
8089}
8090EXPORT_SYMBOL(netdev_refcnt_read);
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104static void netdev_wait_allrefs(struct net_device *dev)
8105{
8106 unsigned long rebroadcast_time, warning_time;
8107 int refcnt;
8108
8109 linkwatch_forget_dev(dev);
8110
8111 rebroadcast_time = warning_time = jiffies;
8112 refcnt = netdev_refcnt_read(dev);
8113
8114 while (refcnt != 0) {
8115 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
8116 rtnl_lock();
8117
8118
8119 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8120
8121 __rtnl_unlock();
8122 rcu_barrier();
8123 rtnl_lock();
8124
8125 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
8126 &dev->state)) {
8127
8128
8129
8130
8131
8132
8133 linkwatch_run_queue();
8134 }
8135
8136 __rtnl_unlock();
8137
8138 rebroadcast_time = jiffies;
8139 }
8140
8141 msleep(250);
8142
8143 refcnt = netdev_refcnt_read(dev);
8144
8145 if (time_after(jiffies, warning_time + 10 * HZ)) {
8146 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
8147 dev->name, refcnt);
8148 warning_time = jiffies;
8149 }
8150 }
8151}
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177void netdev_run_todo(void)
8178{
8179 struct list_head list;
8180
8181
8182 list_replace_init(&net_todo_list, &list);
8183
8184 __rtnl_unlock();
8185
8186
8187
8188 if (!list_empty(&list))
8189 rcu_barrier();
8190
8191 while (!list_empty(&list)) {
8192 struct net_device *dev
8193 = list_first_entry(&list, struct net_device, todo_list);
8194 list_del(&dev->todo_list);
8195
8196 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
8197 pr_err("network todo '%s' but state %d\n",
8198 dev->name, dev->reg_state);
8199 dump_stack();
8200 continue;
8201 }
8202
8203 dev->reg_state = NETREG_UNREGISTERED;
8204
8205 netdev_wait_allrefs(dev);
8206
8207
8208 BUG_ON(netdev_refcnt_read(dev));
8209 BUG_ON(!list_empty(&dev->ptype_all));
8210 BUG_ON(!list_empty(&dev->ptype_specific));
8211 WARN_ON(rcu_access_pointer(dev->ip_ptr));
8212 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
8213#if IS_ENABLED(CONFIG_DECNET)
8214 WARN_ON(dev->dn_ptr);
8215#endif
8216 if (dev->priv_destructor)
8217 dev->priv_destructor(dev);
8218 if (dev->needs_free_netdev)
8219 free_netdev(dev);
8220
8221
8222 rtnl_lock();
8223 dev_net(dev)->dev_unreg_count--;
8224 __rtnl_unlock();
8225 wake_up(&netdev_unregistering_wq);
8226
8227
8228 kobject_put(&dev->dev.kobj);
8229 }
8230}
8231
8232
8233
8234
8235
8236
8237void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
8238 const struct net_device_stats *netdev_stats)
8239{
8240#if BITS_PER_LONG == 64
8241 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
8242 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
8243
8244 memset((char *)stats64 + sizeof(*netdev_stats), 0,
8245 sizeof(*stats64) - sizeof(*netdev_stats));
8246#else
8247 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
8248 const unsigned long *src = (const unsigned long *)netdev_stats;
8249 u64 *dst = (u64 *)stats64;
8250
8251 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
8252 for (i = 0; i < n; i++)
8253 dst[i] = src[i];
8254
8255 memset((char *)stats64 + n * sizeof(u64), 0,
8256 sizeof(*stats64) - n * sizeof(u64));
8257#endif
8258}
8259EXPORT_SYMBOL(netdev_stats_to_stats64);
8260
8261
8262
8263
8264
8265
8266
8267
8268
8269
8270
8271struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
8272 struct rtnl_link_stats64 *storage)
8273{
8274 const struct net_device_ops *ops = dev->netdev_ops;
8275
8276 if (ops->ndo_get_stats64) {
8277 memset(storage, 0, sizeof(*storage));
8278 ops->ndo_get_stats64(dev, storage);
8279 } else if (ops->ndo_get_stats) {
8280 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
8281 } else {
8282 netdev_stats_to_stats64(storage, &dev->stats);
8283 }
8284 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
8285 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
8286 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
8287 return storage;
8288}
8289EXPORT_SYMBOL(dev_get_stats);
8290
8291struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
8292{
8293 struct netdev_queue *queue = dev_ingress_queue(dev);
8294
8295#ifdef CONFIG_NET_CLS_ACT
8296 if (queue)
8297 return queue;
8298 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
8299 if (!queue)
8300 return NULL;
8301 netdev_init_one_queue(dev, queue, NULL);
8302 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
8303 queue->qdisc_sleeping = &noop_qdisc;
8304 rcu_assign_pointer(dev->ingress_queue, queue);
8305#endif
8306 return queue;
8307}
8308
8309static const struct ethtool_ops default_ethtool_ops;
8310
8311void netdev_set_default_ethtool_ops(struct net_device *dev,
8312 const struct ethtool_ops *ops)
8313{
8314 if (dev->ethtool_ops == &default_ethtool_ops)
8315 dev->ethtool_ops = ops;
8316}
8317EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
8318
8319void netdev_freemem(struct net_device *dev)
8320{
8321 char *addr = (char *)dev - dev->padded;
8322
8323 kvfree(addr);
8324}
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
8340 unsigned char name_assign_type,
8341 void (*setup)(struct net_device *),
8342 unsigned int txqs, unsigned int rxqs)
8343{
8344 struct net_device *dev;
8345 unsigned int alloc_size;
8346 struct net_device *p;
8347
8348 BUG_ON(strlen(name) >= sizeof(dev->name));
8349
8350 if (txqs < 1) {
8351 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
8352 return NULL;
8353 }
8354
8355 if (rxqs < 1) {
8356 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
8357 return NULL;
8358 }
8359
8360 alloc_size = sizeof(struct net_device);
8361 if (sizeof_priv) {
8362
8363 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
8364 alloc_size += sizeof_priv;
8365 }
8366
8367 alloc_size += NETDEV_ALIGN - 1;
8368
8369 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
8370 if (!p)
8371 return NULL;
8372
8373 dev = PTR_ALIGN(p, NETDEV_ALIGN);
8374 dev->padded = (char *)dev - (char *)p;
8375
8376 dev->pcpu_refcnt = alloc_percpu(int);
8377 if (!dev->pcpu_refcnt)
8378 goto free_dev;
8379
8380 if (dev_addr_init(dev))
8381 goto free_pcpu;
8382
8383 dev_mc_init(dev);
8384 dev_uc_init(dev);
8385
8386 dev_net_set(dev, &init_net);
8387
8388 dev->gso_max_size = GSO_MAX_SIZE;
8389 dev->gso_max_segs = GSO_MAX_SEGS;
8390
8391 INIT_LIST_HEAD(&dev->napi_list);
8392 INIT_LIST_HEAD(&dev->unreg_list);
8393 INIT_LIST_HEAD(&dev->close_list);
8394 INIT_LIST_HEAD(&dev->link_watch_list);
8395 INIT_LIST_HEAD(&dev->adj_list.upper);
8396 INIT_LIST_HEAD(&dev->adj_list.lower);
8397 INIT_LIST_HEAD(&dev->ptype_all);
8398 INIT_LIST_HEAD(&dev->ptype_specific);
8399#ifdef CONFIG_NET_SCHED
8400 hash_init(dev->qdisc_hash);
8401#endif
8402 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
8403 setup(dev);
8404
8405 if (!dev->tx_queue_len) {
8406 dev->priv_flags |= IFF_NO_QUEUE;
8407 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
8408 }
8409
8410 dev->num_tx_queues = txqs;
8411 dev->real_num_tx_queues = txqs;
8412 if (netif_alloc_netdev_queues(dev))
8413 goto free_all;
8414
8415 dev->num_rx_queues = rxqs;
8416 dev->real_num_rx_queues = rxqs;
8417 if (netif_alloc_rx_queues(dev))
8418 goto free_all;
8419
8420 strcpy(dev->name, name);
8421 dev->name_assign_type = name_assign_type;
8422 dev->group = INIT_NETDEV_GROUP;
8423 if (!dev->ethtool_ops)
8424 dev->ethtool_ops = &default_ethtool_ops;
8425
8426 nf_hook_ingress_init(dev);
8427
8428 return dev;
8429
8430free_all:
8431 free_netdev(dev);
8432 return NULL;
8433
8434free_pcpu:
8435 free_percpu(dev->pcpu_refcnt);
8436free_dev:
8437 netdev_freemem(dev);
8438 return NULL;
8439}
8440EXPORT_SYMBOL(alloc_netdev_mqs);
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451void free_netdev(struct net_device *dev)
8452{
8453 struct napi_struct *p, *n;
8454
8455 might_sleep();
8456 netif_free_tx_queues(dev);
8457 netif_free_rx_queues(dev);
8458
8459 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
8460
8461
8462 dev_addr_flush(dev);
8463
8464 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
8465 netif_napi_del(p);
8466
8467 free_percpu(dev->pcpu_refcnt);
8468 dev->pcpu_refcnt = NULL;
8469
8470
8471 if (dev->reg_state == NETREG_UNINITIALIZED) {
8472 netdev_freemem(dev);
8473 return;
8474 }
8475
8476 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
8477 dev->reg_state = NETREG_RELEASED;
8478
8479
8480 put_device(&dev->dev);
8481}
8482EXPORT_SYMBOL(free_netdev);
8483
8484
8485
8486
8487
8488
8489
8490void synchronize_net(void)
8491{
8492 might_sleep();
8493 if (rtnl_is_locked())
8494 synchronize_rcu_expedited();
8495 else
8496 synchronize_rcu();
8497}
8498EXPORT_SYMBOL(synchronize_net);
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
8514{
8515 ASSERT_RTNL();
8516
8517 if (head) {
8518 list_move_tail(&dev->unreg_list, head);
8519 } else {
8520 rollback_registered(dev);
8521
8522 net_set_todo(dev);
8523 }
8524}
8525EXPORT_SYMBOL(unregister_netdevice_queue);
8526
8527
8528
8529
8530
8531
8532
8533
8534void unregister_netdevice_many(struct list_head *head)
8535{
8536 struct net_device *dev;
8537
8538 if (!list_empty(head)) {
8539 rollback_registered_many(head);
8540 list_for_each_entry(dev, head, unreg_list)
8541 net_set_todo(dev);
8542 list_del(head);
8543 }
8544}
8545EXPORT_SYMBOL(unregister_netdevice_many);
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558void unregister_netdev(struct net_device *dev)
8559{
8560 rtnl_lock();
8561 unregister_netdevice(dev);
8562 rtnl_unlock();
8563}
8564EXPORT_SYMBOL(unregister_netdev);
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
8581{
8582 int err, new_nsid, new_ifindex;
8583
8584 ASSERT_RTNL();
8585
8586
8587 err = -EINVAL;
8588 if (dev->features & NETIF_F_NETNS_LOCAL)
8589 goto out;
8590
8591
8592 if (dev->reg_state != NETREG_REGISTERED)
8593 goto out;
8594
8595
8596 err = 0;
8597 if (net_eq(dev_net(dev), net))
8598 goto out;
8599
8600
8601
8602
8603 err = -EEXIST;
8604 if (__dev_get_by_name(net, dev->name)) {
8605
8606 if (!pat)
8607 goto out;
8608 if (dev_get_valid_name(net, dev, pat) < 0)
8609 goto out;
8610 }
8611
8612
8613
8614
8615
8616
8617 dev_close(dev);
8618
8619
8620 err = -ENODEV;
8621 unlist_netdevice(dev);
8622
8623 synchronize_net();
8624
8625
8626 dev_shutdown(dev);
8627
8628
8629
8630
8631
8632
8633
8634
8635 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8636 rcu_barrier();
8637
8638 new_nsid = peernet2id_alloc(dev_net(dev), net);
8639
8640 if (__dev_get_by_index(net, dev->ifindex))
8641 new_ifindex = dev_new_index(net);
8642 else
8643 new_ifindex = dev->ifindex;
8644
8645 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
8646 new_ifindex);
8647
8648
8649
8650
8651 dev_uc_flush(dev);
8652 dev_mc_flush(dev);
8653
8654
8655 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
8656 netdev_adjacent_del_links(dev);
8657
8658
8659 dev_net_set(dev, net);
8660 dev->ifindex = new_ifindex;
8661
8662
8663 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
8664 netdev_adjacent_add_links(dev);
8665
8666
8667 err = device_rename(&dev->dev, dev->name);
8668 WARN_ON(err);
8669
8670
8671 list_netdevice(dev);
8672
8673
8674 call_netdevice_notifiers(NETDEV_REGISTER, dev);
8675
8676
8677
8678
8679
8680 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
8681
8682 synchronize_net();
8683 err = 0;
8684out:
8685 return err;
8686}
8687EXPORT_SYMBOL_GPL(dev_change_net_namespace);
8688
8689static int dev_cpu_dead(unsigned int oldcpu)
8690{
8691 struct sk_buff **list_skb;
8692 struct sk_buff *skb;
8693 unsigned int cpu;
8694 struct softnet_data *sd, *oldsd, *remsd = NULL;
8695
8696 local_irq_disable();
8697 cpu = smp_processor_id();
8698 sd = &per_cpu(softnet_data, cpu);
8699 oldsd = &per_cpu(softnet_data, oldcpu);
8700
8701
8702 list_skb = &sd->completion_queue;
8703 while (*list_skb)
8704 list_skb = &(*list_skb)->next;
8705
8706 *list_skb = oldsd->completion_queue;
8707 oldsd->completion_queue = NULL;
8708
8709
8710 if (oldsd->output_queue) {
8711 *sd->output_queue_tailp = oldsd->output_queue;
8712 sd->output_queue_tailp = oldsd->output_queue_tailp;
8713 oldsd->output_queue = NULL;
8714 oldsd->output_queue_tailp = &oldsd->output_queue;
8715 }
8716
8717
8718
8719
8720 while (!list_empty(&oldsd->poll_list)) {
8721 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8722 struct napi_struct,
8723 poll_list);
8724
8725 list_del_init(&napi->poll_list);
8726 if (napi->poll == process_backlog)
8727 napi->state = 0;
8728 else
8729 ____napi_schedule(sd, napi);
8730 }
8731
8732 raise_softirq_irqoff(NET_TX_SOFTIRQ);
8733 local_irq_enable();
8734
8735#ifdef CONFIG_RPS
8736 remsd = oldsd->rps_ipi_list;
8737 oldsd->rps_ipi_list = NULL;
8738#endif
8739
8740 net_rps_send_ipi(remsd);
8741
8742
8743 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
8744 netif_rx_ni(skb);
8745 input_queue_head_incr(oldsd);
8746 }
8747 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
8748 netif_rx_ni(skb);
8749 input_queue_head_incr(oldsd);
8750 }
8751
8752 return 0;
8753}
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765netdev_features_t netdev_increment_features(netdev_features_t all,
8766 netdev_features_t one, netdev_features_t mask)
8767{
8768 if (mask & NETIF_F_HW_CSUM)
8769 mask |= NETIF_F_CSUM_MASK;
8770 mask |= NETIF_F_VLAN_CHALLENGED;
8771
8772 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
8773 all &= one | ~NETIF_F_ALL_FOR_ALL;
8774
8775
8776 if (all & NETIF_F_HW_CSUM)
8777 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
8778
8779 return all;
8780}
8781EXPORT_SYMBOL(netdev_increment_features);
8782
8783static struct hlist_head * __net_init netdev_create_hash(void)
8784{
8785 int i;
8786 struct hlist_head *hash;
8787
8788 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8789 if (hash != NULL)
8790 for (i = 0; i < NETDEV_HASHENTRIES; i++)
8791 INIT_HLIST_HEAD(&hash[i]);
8792
8793 return hash;
8794}
8795
8796
8797static int __net_init netdev_init(struct net *net)
8798{
8799 if (net != &init_net)
8800 INIT_LIST_HEAD(&net->dev_base_head);
8801
8802 net->dev_name_head = netdev_create_hash();
8803 if (net->dev_name_head == NULL)
8804 goto err_name;
8805
8806 net->dev_index_head = netdev_create_hash();
8807 if (net->dev_index_head == NULL)
8808 goto err_idx;
8809
8810 return 0;
8811
8812err_idx:
8813 kfree(net->dev_name_head);
8814err_name:
8815 return -ENOMEM;
8816}
8817
8818
8819
8820
8821
8822
8823
8824const char *netdev_drivername(const struct net_device *dev)
8825{
8826 const struct device_driver *driver;
8827 const struct device *parent;
8828 const char *empty = "";
8829
8830 parent = dev->dev.parent;
8831 if (!parent)
8832 return empty;
8833
8834 driver = parent->driver;
8835 if (driver && driver->name)
8836 return driver->name;
8837 return empty;
8838}
8839
8840static void __netdev_printk(const char *level, const struct net_device *dev,
8841 struct va_format *vaf)
8842{
8843 if (dev && dev->dev.parent) {
8844 dev_printk_emit(level[1] - '0',
8845 dev->dev.parent,
8846 "%s %s %s%s: %pV",
8847 dev_driver_string(dev->dev.parent),
8848 dev_name(dev->dev.parent),
8849 netdev_name(dev), netdev_reg_state(dev),
8850 vaf);
8851 } else if (dev) {
8852 printk("%s%s%s: %pV",
8853 level, netdev_name(dev), netdev_reg_state(dev), vaf);
8854 } else {
8855 printk("%s(NULL net_device): %pV", level, vaf);
8856 }
8857}
8858
8859void netdev_printk(const char *level, const struct net_device *dev,
8860 const char *format, ...)
8861{
8862 struct va_format vaf;
8863 va_list args;
8864
8865 va_start(args, format);
8866
8867 vaf.fmt = format;
8868 vaf.va = &args;
8869
8870 __netdev_printk(level, dev, &vaf);
8871
8872 va_end(args);
8873}
8874EXPORT_SYMBOL(netdev_printk);
8875
8876#define define_netdev_printk_level(func, level) \
8877void func(const struct net_device *dev, const char *fmt, ...) \
8878{ \
8879 struct va_format vaf; \
8880 va_list args; \
8881 \
8882 va_start(args, fmt); \
8883 \
8884 vaf.fmt = fmt; \
8885 vaf.va = &args; \
8886 \
8887 __netdev_printk(level, dev, &vaf); \
8888 \
8889 va_end(args); \
8890} \
8891EXPORT_SYMBOL(func);
8892
8893define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8894define_netdev_printk_level(netdev_alert, KERN_ALERT);
8895define_netdev_printk_level(netdev_crit, KERN_CRIT);
8896define_netdev_printk_level(netdev_err, KERN_ERR);
8897define_netdev_printk_level(netdev_warn, KERN_WARNING);
8898define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8899define_netdev_printk_level(netdev_info, KERN_INFO);
8900
8901static void __net_exit netdev_exit(struct net *net)
8902{
8903 kfree(net->dev_name_head);
8904 kfree(net->dev_index_head);
8905 if (net != &init_net)
8906 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
8907}
8908
8909static struct pernet_operations __net_initdata netdev_net_ops = {
8910 .init = netdev_init,
8911 .exit = netdev_exit,
8912};
8913
8914static void __net_exit default_device_exit(struct net *net)
8915{
8916 struct net_device *dev, *aux;
8917
8918
8919
8920
8921 rtnl_lock();
8922 for_each_netdev_safe(net, dev, aux) {
8923 int err;
8924 char fb_name[IFNAMSIZ];
8925
8926
8927 if (dev->features & NETIF_F_NETNS_LOCAL)
8928 continue;
8929
8930
8931 if (dev->rtnl_link_ops)
8932 continue;
8933
8934
8935 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8936 err = dev_change_net_namespace(dev, &init_net, fb_name);
8937 if (err) {
8938 pr_emerg("%s: failed to move %s to init_net: %d\n",
8939 __func__, dev->name, err);
8940 BUG();
8941 }
8942 }
8943 rtnl_unlock();
8944}
8945
8946static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8947{
8948
8949
8950
8951 struct net *net;
8952 bool unregistering;
8953 DEFINE_WAIT_FUNC(wait, woken_wake_function);
8954
8955 add_wait_queue(&netdev_unregistering_wq, &wait);
8956 for (;;) {
8957 unregistering = false;
8958 rtnl_lock();
8959 list_for_each_entry(net, net_list, exit_list) {
8960 if (net->dev_unreg_count > 0) {
8961 unregistering = true;
8962 break;
8963 }
8964 }
8965 if (!unregistering)
8966 break;
8967 __rtnl_unlock();
8968
8969 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8970 }
8971 remove_wait_queue(&netdev_unregistering_wq, &wait);
8972}
8973
8974static void __net_exit default_device_exit_batch(struct list_head *net_list)
8975{
8976
8977
8978
8979
8980
8981 struct net_device *dev;
8982 struct net *net;
8983 LIST_HEAD(dev_kill_list);
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996 rtnl_lock_unregistering(net_list);
8997 list_for_each_entry(net, net_list, exit_list) {
8998 for_each_netdev_reverse(net, dev) {
8999 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
9000 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
9001 else
9002 unregister_netdevice_queue(dev, &dev_kill_list);
9003 }
9004 }
9005 unregister_netdevice_many(&dev_kill_list);
9006 rtnl_unlock();
9007}
9008
9009static struct pernet_operations __net_initdata default_device_ops = {
9010 .exit = default_device_exit,
9011 .exit_batch = default_device_exit_batch,
9012};
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025static int __init net_dev_init(void)
9026{
9027 int i, rc = -ENOMEM;
9028
9029 BUG_ON(!dev_boot_phase);
9030
9031 if (dev_proc_init())
9032 goto out;
9033
9034 if (netdev_kobject_init())
9035 goto out;
9036
9037 INIT_LIST_HEAD(&ptype_all);
9038 for (i = 0; i < PTYPE_HASH_SIZE; i++)
9039 INIT_LIST_HEAD(&ptype_base[i]);
9040
9041 INIT_LIST_HEAD(&offload_base);
9042
9043 if (register_pernet_subsys(&netdev_net_ops))
9044 goto out;
9045
9046
9047
9048
9049
9050 for_each_possible_cpu(i) {
9051 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
9052 struct softnet_data *sd = &per_cpu(softnet_data, i);
9053
9054 INIT_WORK(flush, flush_backlog);
9055
9056 skb_queue_head_init(&sd->input_pkt_queue);
9057 skb_queue_head_init(&sd->process_queue);
9058#ifdef CONFIG_XFRM_OFFLOAD
9059 skb_queue_head_init(&sd->xfrm_backlog);
9060#endif
9061 INIT_LIST_HEAD(&sd->poll_list);
9062 sd->output_queue_tailp = &sd->output_queue;
9063#ifdef CONFIG_RPS
9064 sd->csd.func = rps_trigger_softirq;
9065 sd->csd.info = sd;
9066 sd->cpu = i;
9067#endif
9068
9069 sd->backlog.poll = process_backlog;
9070 sd->backlog.weight = weight_p;
9071 }
9072
9073 dev_boot_phase = 0;
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084 if (register_pernet_device(&loopback_net_ops))
9085 goto out;
9086
9087 if (register_pernet_device(&default_device_ops))
9088 goto out;
9089
9090 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
9091 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
9092
9093 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
9094 NULL, dev_cpu_dead);
9095 WARN_ON(rc < 0);
9096 rc = 0;
9097out:
9098 return rc;
9099}
9100
9101subsys_initcall(net_dev_init);
9102