1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef _LINUX_NETDEVICE_H
26#define _LINUX_NETDEVICE_H
27
28#include <linux/timer.h>
29#include <linux/bug.h>
30#include <linux/delay.h>
31#include <linux/atomic.h>
32#include <linux/prefetch.h>
33#include <asm/cache.h>
34#include <asm/byteorder.h>
35
36#include <linux/percpu.h>
37#include <linux/rculist.h>
38#include <linux/workqueue.h>
39#include <linux/dynamic_queue_limits.h>
40
41#include <linux/ethtool.h>
42#include <net/net_namespace.h>
43#ifdef CONFIG_DCB
44#include <net/dcbnl.h>
45#endif
46#include <net/netprio_cgroup.h>
47#include <net/xdp.h>
48
49#include <linux/netdev_features.h>
50#include <linux/neighbour.h>
51#include <uapi/linux/netdevice.h>
52#include <uapi/linux/if_bonding.h>
53#include <uapi/linux/pkt_cls.h>
54#include <linux/hashtable.h>
55
56struct netpoll_info;
57struct device;
58struct phy_device;
59struct dsa_port;
60
61struct sfp_bus;
62
63struct wireless_dev;
64
65struct wpan_dev;
66struct mpls_dev;
67
68struct udp_tunnel_info;
69struct bpf_prog;
70struct xdp_buff;
71
72void netdev_set_default_ethtool_ops(struct net_device *dev,
73 const struct ethtool_ops *ops);
74
75
76#define NET_RX_SUCCESS 0
77#define NET_RX_DROP 1
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97#define NET_XMIT_SUCCESS 0x00
98#define NET_XMIT_DROP 0x01
99#define NET_XMIT_CN 0x02
100#define NET_XMIT_MASK 0x0f
101
102
103
104
105#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
106#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
107
108
109#define NETDEV_TX_MASK 0xf0
110
111enum netdev_tx {
112 __NETDEV_TX_MIN = INT_MIN,
113 NETDEV_TX_OK = 0x00,
114 NETDEV_TX_BUSY = 0x10,
115};
116typedef enum netdev_tx netdev_tx_t;
117
118
119
120
121
122static inline bool dev_xmit_complete(int rc)
123{
124
125
126
127
128
129
130 if (likely(rc < NET_XMIT_MASK))
131 return true;
132
133 return false;
134}
135
136
137
138
139
140
141#if defined(CONFIG_HYPERV_NET)
142# define LL_MAX_HEADER 128
143#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
144# if defined(CONFIG_MAC80211_MESH)
145# define LL_MAX_HEADER 128
146# else
147# define LL_MAX_HEADER 96
148# endif
149#else
150# define LL_MAX_HEADER 32
151#endif
152
153#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
154 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
155#define MAX_HEADER LL_MAX_HEADER
156#else
157#define MAX_HEADER (LL_MAX_HEADER + 48)
158#endif
159
160
161
162
163
164
165struct net_device_stats {
166 unsigned long rx_packets;
167 unsigned long tx_packets;
168 unsigned long rx_bytes;
169 unsigned long tx_bytes;
170 unsigned long rx_errors;
171 unsigned long tx_errors;
172 unsigned long rx_dropped;
173 unsigned long tx_dropped;
174 unsigned long multicast;
175 unsigned long collisions;
176 unsigned long rx_length_errors;
177 unsigned long rx_over_errors;
178 unsigned long rx_crc_errors;
179 unsigned long rx_frame_errors;
180 unsigned long rx_fifo_errors;
181 unsigned long rx_missed_errors;
182 unsigned long tx_aborted_errors;
183 unsigned long tx_carrier_errors;
184 unsigned long tx_fifo_errors;
185 unsigned long tx_heartbeat_errors;
186 unsigned long tx_window_errors;
187 unsigned long rx_compressed;
188 unsigned long tx_compressed;
189};
190
191
192#include <linux/cache.h>
193#include <linux/skbuff.h>
194
195#ifdef CONFIG_RPS
196#include <linux/static_key.h>
197extern struct static_key rps_needed;
198extern struct static_key rfs_needed;
199#endif
200
201struct neighbour;
202struct neigh_parms;
203struct sk_buff;
204
205struct netdev_hw_addr {
206 struct list_head list;
207 unsigned char addr[MAX_ADDR_LEN];
208 unsigned char type;
209#define NETDEV_HW_ADDR_T_LAN 1
210#define NETDEV_HW_ADDR_T_SAN 2
211#define NETDEV_HW_ADDR_T_SLAVE 3
212#define NETDEV_HW_ADDR_T_UNICAST 4
213#define NETDEV_HW_ADDR_T_MULTICAST 5
214 bool global_use;
215 int sync_cnt;
216 int refcount;
217 int synced;
218 struct rcu_head rcu_head;
219};
220
221struct netdev_hw_addr_list {
222 struct list_head list;
223 int count;
224};
225
226#define netdev_hw_addr_list_count(l) ((l)->count)
227#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
228#define netdev_hw_addr_list_for_each(ha, l) \
229 list_for_each_entry(ha, &(l)->list, list)
230
231#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
232#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
233#define netdev_for_each_uc_addr(ha, dev) \
234 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
235
236#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
237#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
238#define netdev_for_each_mc_addr(ha, dev) \
239 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
240
241struct hh_cache {
242 unsigned int hh_len;
243 seqlock_t hh_lock;
244
245
246#define HH_DATA_MOD 16
247#define HH_DATA_OFF(__len) \
248 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
249#define HH_DATA_ALIGN(__len) \
250 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
251 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
252};
253
254
255
256
257
258
259
260
261
262#define LL_RESERVED_SPACE(dev) \
263 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
264#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
265 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
266
267struct header_ops {
268 int (*create) (struct sk_buff *skb, struct net_device *dev,
269 unsigned short type, const void *daddr,
270 const void *saddr, unsigned int len);
271 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
272 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
273 void (*cache_update)(struct hh_cache *hh,
274 const struct net_device *dev,
275 const unsigned char *haddr);
276 bool (*validate)(const char *ll_header, unsigned int len);
277};
278
279
280
281
282
283
284enum netdev_state_t {
285 __LINK_STATE_START,
286 __LINK_STATE_PRESENT,
287 __LINK_STATE_NOCARRIER,
288 __LINK_STATE_LINKWATCH_PENDING,
289 __LINK_STATE_DORMANT,
290};
291
292
293
294
295
296
297struct netdev_boot_setup {
298 char name[IFNAMSIZ];
299 struct ifmap map;
300};
301#define NETDEV_BOOT_SETUP_MAX 8
302
303int __init netdev_boot_setup(char *str);
304
305
306
307
308struct napi_struct {
309
310
311
312
313
314
315 struct list_head poll_list;
316
317 unsigned long state;
318 int weight;
319 unsigned int gro_count;
320 int (*poll)(struct napi_struct *, int);
321#ifdef CONFIG_NETPOLL
322 int poll_owner;
323#endif
324 struct net_device *dev;
325 struct sk_buff *gro_list;
326 struct sk_buff *skb;
327 struct hrtimer timer;
328 struct list_head dev_list;
329 struct hlist_node napi_hash_node;
330 unsigned int napi_id;
331};
332
333enum {
334 NAPI_STATE_SCHED,
335 NAPI_STATE_MISSED,
336 NAPI_STATE_DISABLE,
337 NAPI_STATE_NPSVC,
338 NAPI_STATE_HASHED,
339 NAPI_STATE_NO_BUSY_POLL,
340 NAPI_STATE_IN_BUSY_POLL,
341};
342
343enum {
344 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
345 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
346 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
347 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
348 NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED),
349 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
350 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
351};
352
353enum gro_result {
354 GRO_MERGED,
355 GRO_MERGED_FREE,
356 GRO_HELD,
357 GRO_NORMAL,
358 GRO_DROP,
359 GRO_CONSUMED,
360};
361typedef enum gro_result gro_result_t;
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404enum rx_handler_result {
405 RX_HANDLER_CONSUMED,
406 RX_HANDLER_ANOTHER,
407 RX_HANDLER_EXACT,
408 RX_HANDLER_PASS,
409};
410typedef enum rx_handler_result rx_handler_result_t;
411typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
412
413void __napi_schedule(struct napi_struct *n);
414void __napi_schedule_irqoff(struct napi_struct *n);
415
416static inline bool napi_disable_pending(struct napi_struct *n)
417{
418 return test_bit(NAPI_STATE_DISABLE, &n->state);
419}
420
421bool napi_schedule_prep(struct napi_struct *n);
422
423
424
425
426
427
428
429
430static inline void napi_schedule(struct napi_struct *n)
431{
432 if (napi_schedule_prep(n))
433 __napi_schedule(n);
434}
435
436
437
438
439
440
441
442static inline void napi_schedule_irqoff(struct napi_struct *n)
443{
444 if (napi_schedule_prep(n))
445 __napi_schedule_irqoff(n);
446}
447
448
449static inline bool napi_reschedule(struct napi_struct *napi)
450{
451 if (napi_schedule_prep(napi)) {
452 __napi_schedule(napi);
453 return true;
454 }
455 return false;
456}
457
458bool napi_complete_done(struct napi_struct *n, int work_done);
459
460
461
462
463
464
465
466
467static inline bool napi_complete(struct napi_struct *n)
468{
469 return napi_complete_done(n, 0);
470}
471
472
473
474
475
476
477
478
479
480
481
482
483
484bool napi_hash_del(struct napi_struct *napi);
485
486
487
488
489
490
491
492
493void napi_disable(struct napi_struct *n);
494
495
496
497
498
499
500
501
502static inline void napi_enable(struct napi_struct *n)
503{
504 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
505 smp_mb__before_atomic();
506 clear_bit(NAPI_STATE_SCHED, &n->state);
507 clear_bit(NAPI_STATE_NPSVC, &n->state);
508}
509
510
511
512
513
514
515
516
517
518static inline void napi_synchronize(const struct napi_struct *n)
519{
520 if (IS_ENABLED(CONFIG_SMP))
521 while (test_bit(NAPI_STATE_SCHED, &n->state))
522 msleep(1);
523 else
524 barrier();
525}
526
527enum netdev_queue_state_t {
528 __QUEUE_STATE_DRV_XOFF,
529 __QUEUE_STATE_STACK_XOFF,
530 __QUEUE_STATE_FROZEN,
531};
532
533#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
534#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
535#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
536
537#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
538#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
539 QUEUE_STATE_FROZEN)
540#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
541 QUEUE_STATE_FROZEN)
542
543
544
545
546
547
548
549
550
551
552
553struct netdev_queue {
554
555
556
557 struct net_device *dev;
558 struct Qdisc __rcu *qdisc;
559 struct Qdisc *qdisc_sleeping;
560#ifdef CONFIG_SYSFS
561 struct kobject kobj;
562#endif
563#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
564 int numa_node;
565#endif
566 unsigned long tx_maxrate;
567
568
569
570
571 unsigned long trans_timeout;
572
573
574
575 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
576 int xmit_lock_owner;
577
578
579
580 unsigned long trans_start;
581
582 unsigned long state;
583
584#ifdef CONFIG_BQL
585 struct dql dql;
586#endif
587} ____cacheline_aligned_in_smp;
588
589extern int sysctl_fb_tunnels_only_for_init_net;
590
591static inline bool net_has_fallback_tunnels(const struct net *net)
592{
593 return net == &init_net ||
594 !IS_ENABLED(CONFIG_SYSCTL) ||
595 !sysctl_fb_tunnels_only_for_init_net;
596}
597
598static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
599{
600#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
601 return q->numa_node;
602#else
603 return NUMA_NO_NODE;
604#endif
605}
606
607static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
608{
609#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
610 q->numa_node = node;
611#endif
612}
613
614#ifdef CONFIG_RPS
615
616
617
618
619struct rps_map {
620 unsigned int len;
621 struct rcu_head rcu;
622 u16 cpus[0];
623};
624#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
625
626
627
628
629
630
631struct rps_dev_flow {
632 u16 cpu;
633 u16 filter;
634 unsigned int last_qtail;
635};
636#define RPS_NO_FILTER 0xffff
637
638
639
640
641struct rps_dev_flow_table {
642 unsigned int mask;
643 struct rcu_head rcu;
644 struct rps_dev_flow flows[0];
645};
646#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
647 ((_num) * sizeof(struct rps_dev_flow)))
648
649
650
651
652
653
654
655
656
657
658
659struct rps_sock_flow_table {
660 u32 mask;
661
662 u32 ents[0] ____cacheline_aligned_in_smp;
663};
664#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
665
666#define RPS_NO_CPU 0xffff
667
668extern u32 rps_cpu_mask;
669extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
670
671static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
672 u32 hash)
673{
674 if (table && hash) {
675 unsigned int index = hash & table->mask;
676 u32 val = hash & ~rps_cpu_mask;
677
678
679 val |= raw_smp_processor_id();
680
681 if (table->ents[index] != val)
682 table->ents[index] = val;
683 }
684}
685
686#ifdef CONFIG_RFS_ACCEL
687bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
688 u16 filter_id);
689#endif
690#endif
691
692
693struct netdev_rx_queue {
694#ifdef CONFIG_RPS
695 struct rps_map __rcu *rps_map;
696 struct rps_dev_flow_table __rcu *rps_flow_table;
697#endif
698 struct kobject kobj;
699 struct net_device *dev;
700 struct xdp_rxq_info xdp_rxq;
701} ____cacheline_aligned_in_smp;
702
703
704
705
706struct rx_queue_attribute {
707 struct attribute attr;
708 ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
709 ssize_t (*store)(struct netdev_rx_queue *queue,
710 const char *buf, size_t len);
711};
712
713#ifdef CONFIG_XPS
714
715
716
717
718struct xps_map {
719 unsigned int len;
720 unsigned int alloc_len;
721 struct rcu_head rcu;
722 u16 queues[0];
723};
724#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
725#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
726 - sizeof(struct xps_map)) / sizeof(u16))
727
728
729
730
731struct xps_dev_maps {
732 struct rcu_head rcu;
733 struct xps_map __rcu *cpu_map[0];
734};
735#define XPS_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
736 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
737#endif
738
739#define TC_MAX_QUEUE 16
740#define TC_BITMASK 15
741
742struct netdev_tc_txq {
743 u16 count;
744 u16 offset;
745};
746
747#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
748
749
750
751
752struct netdev_fcoe_hbainfo {
753 char manufacturer[64];
754 char serial_number[64];
755 char hardware_version[64];
756 char driver_version[64];
757 char optionrom_version[64];
758 char firmware_version[64];
759 char model[256];
760 char model_description[256];
761};
762#endif
763
764#define MAX_PHYS_ITEM_ID_LEN 32
765
766
767
768
769struct netdev_phys_item_id {
770 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
771 unsigned char id_len;
772};
773
774static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
775 struct netdev_phys_item_id *b)
776{
777 return a->id_len == b->id_len &&
778 memcmp(a->id, b->id, a->id_len) == 0;
779}
780
781typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
782 struct sk_buff *skb);
783
784enum tc_setup_type {
785 TC_SETUP_QDISC_MQPRIO,
786 TC_SETUP_CLSU32,
787 TC_SETUP_CLSFLOWER,
788 TC_SETUP_CLSMATCHALL,
789 TC_SETUP_CLSBPF,
790 TC_SETUP_BLOCK,
791 TC_SETUP_QDISC_CBS,
792 TC_SETUP_QDISC_RED,
793 TC_SETUP_QDISC_PRIO,
794};
795
796
797
798
799enum bpf_netdev_command {
800
801
802
803
804
805
806
807 XDP_SETUP_PROG,
808 XDP_SETUP_PROG_HW,
809
810
811
812
813 XDP_QUERY_PROG,
814
815 BPF_OFFLOAD_VERIFIER_PREP,
816 BPF_OFFLOAD_TRANSLATE,
817 BPF_OFFLOAD_DESTROY,
818 BPF_OFFLOAD_MAP_ALLOC,
819 BPF_OFFLOAD_MAP_FREE,
820};
821
822struct bpf_prog_offload_ops;
823struct netlink_ext_ack;
824
825struct netdev_bpf {
826 enum bpf_netdev_command command;
827 union {
828
829 struct {
830 u32 flags;
831 struct bpf_prog *prog;
832 struct netlink_ext_ack *extack;
833 };
834
835 struct {
836 u8 prog_attached;
837 u32 prog_id;
838
839 u32 prog_flags;
840 };
841
842 struct {
843 struct bpf_prog *prog;
844 const struct bpf_prog_offload_ops *ops;
845 } verifier;
846
847 struct {
848 struct bpf_prog *prog;
849 } offload;
850
851 struct {
852 struct bpf_offloaded_map *offmap;
853 };
854 };
855};
856
857#ifdef CONFIG_XFRM_OFFLOAD
858struct xfrmdev_ops {
859 int (*xdo_dev_state_add) (struct xfrm_state *x);
860 void (*xdo_dev_state_delete) (struct xfrm_state *x);
861 void (*xdo_dev_state_free) (struct xfrm_state *x);
862 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
863 struct xfrm_state *x);
864 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
865};
866#endif
867
868struct dev_ifalias {
869 struct rcu_head rcuhead;
870 char ifalias[];
871};
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175struct net_device_ops {
1176 int (*ndo_init)(struct net_device *dev);
1177 void (*ndo_uninit)(struct net_device *dev);
1178 int (*ndo_open)(struct net_device *dev);
1179 int (*ndo_stop)(struct net_device *dev);
1180 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1181 struct net_device *dev);
1182 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1183 struct net_device *dev,
1184 netdev_features_t features);
1185 u16 (*ndo_select_queue)(struct net_device *dev,
1186 struct sk_buff *skb,
1187 void *accel_priv,
1188 select_queue_fallback_t fallback);
1189 void (*ndo_change_rx_flags)(struct net_device *dev,
1190 int flags);
1191 void (*ndo_set_rx_mode)(struct net_device *dev);
1192 int (*ndo_set_mac_address)(struct net_device *dev,
1193 void *addr);
1194 int (*ndo_validate_addr)(struct net_device *dev);
1195 int (*ndo_do_ioctl)(struct net_device *dev,
1196 struct ifreq *ifr, int cmd);
1197 int (*ndo_set_config)(struct net_device *dev,
1198 struct ifmap *map);
1199 int (*ndo_change_mtu)(struct net_device *dev,
1200 int new_mtu);
1201 int (*ndo_neigh_setup)(struct net_device *dev,
1202 struct neigh_parms *);
1203 void (*ndo_tx_timeout) (struct net_device *dev);
1204
1205 void (*ndo_get_stats64)(struct net_device *dev,
1206 struct rtnl_link_stats64 *storage);
1207 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1208 int (*ndo_get_offload_stats)(int attr_id,
1209 const struct net_device *dev,
1210 void *attr_data);
1211 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1212
1213 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1214 __be16 proto, u16 vid);
1215 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1216 __be16 proto, u16 vid);
1217#ifdef CONFIG_NET_POLL_CONTROLLER
1218 void (*ndo_poll_controller)(struct net_device *dev);
1219 int (*ndo_netpoll_setup)(struct net_device *dev,
1220 struct netpoll_info *info);
1221 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1222#endif
1223 int (*ndo_set_vf_mac)(struct net_device *dev,
1224 int queue, u8 *mac);
1225 int (*ndo_set_vf_vlan)(struct net_device *dev,
1226 int queue, u16 vlan,
1227 u8 qos, __be16 proto);
1228 int (*ndo_set_vf_rate)(struct net_device *dev,
1229 int vf, int min_tx_rate,
1230 int max_tx_rate);
1231 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1232 int vf, bool setting);
1233 int (*ndo_set_vf_trust)(struct net_device *dev,
1234 int vf, bool setting);
1235 int (*ndo_get_vf_config)(struct net_device *dev,
1236 int vf,
1237 struct ifla_vf_info *ivf);
1238 int (*ndo_set_vf_link_state)(struct net_device *dev,
1239 int vf, int link_state);
1240 int (*ndo_get_vf_stats)(struct net_device *dev,
1241 int vf,
1242 struct ifla_vf_stats
1243 *vf_stats);
1244 int (*ndo_set_vf_port)(struct net_device *dev,
1245 int vf,
1246 struct nlattr *port[]);
1247 int (*ndo_get_vf_port)(struct net_device *dev,
1248 int vf, struct sk_buff *skb);
1249 int (*ndo_set_vf_guid)(struct net_device *dev,
1250 int vf, u64 guid,
1251 int guid_type);
1252 int (*ndo_set_vf_rss_query_en)(
1253 struct net_device *dev,
1254 int vf, bool setting);
1255 int (*ndo_setup_tc)(struct net_device *dev,
1256 enum tc_setup_type type,
1257 void *type_data);
1258#if IS_ENABLED(CONFIG_FCOE)
1259 int (*ndo_fcoe_enable)(struct net_device *dev);
1260 int (*ndo_fcoe_disable)(struct net_device *dev);
1261 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1262 u16 xid,
1263 struct scatterlist *sgl,
1264 unsigned int sgc);
1265 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1266 u16 xid);
1267 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1268 u16 xid,
1269 struct scatterlist *sgl,
1270 unsigned int sgc);
1271 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1272 struct netdev_fcoe_hbainfo *hbainfo);
1273#endif
1274
1275#if IS_ENABLED(CONFIG_LIBFCOE)
1276#define NETDEV_FCOE_WWNN 0
1277#define NETDEV_FCOE_WWPN 1
1278 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1279 u64 *wwn, int type);
1280#endif
1281
1282#ifdef CONFIG_RFS_ACCEL
1283 int (*ndo_rx_flow_steer)(struct net_device *dev,
1284 const struct sk_buff *skb,
1285 u16 rxq_index,
1286 u32 flow_id);
1287#endif
1288 int (*ndo_add_slave)(struct net_device *dev,
1289 struct net_device *slave_dev,
1290 struct netlink_ext_ack *extack);
1291 int (*ndo_del_slave)(struct net_device *dev,
1292 struct net_device *slave_dev);
1293 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1294 netdev_features_t features);
1295 int (*ndo_set_features)(struct net_device *dev,
1296 netdev_features_t features);
1297 int (*ndo_neigh_construct)(struct net_device *dev,
1298 struct neighbour *n);
1299 void (*ndo_neigh_destroy)(struct net_device *dev,
1300 struct neighbour *n);
1301
1302 int (*ndo_fdb_add)(struct ndmsg *ndm,
1303 struct nlattr *tb[],
1304 struct net_device *dev,
1305 const unsigned char *addr,
1306 u16 vid,
1307 u16 flags);
1308 int (*ndo_fdb_del)(struct ndmsg *ndm,
1309 struct nlattr *tb[],
1310 struct net_device *dev,
1311 const unsigned char *addr,
1312 u16 vid);
1313 int (*ndo_fdb_dump)(struct sk_buff *skb,
1314 struct netlink_callback *cb,
1315 struct net_device *dev,
1316 struct net_device *filter_dev,
1317 int *idx);
1318
1319 int (*ndo_bridge_setlink)(struct net_device *dev,
1320 struct nlmsghdr *nlh,
1321 u16 flags);
1322 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1323 u32 pid, u32 seq,
1324 struct net_device *dev,
1325 u32 filter_mask,
1326 int nlflags);
1327 int (*ndo_bridge_dellink)(struct net_device *dev,
1328 struct nlmsghdr *nlh,
1329 u16 flags);
1330 int (*ndo_change_carrier)(struct net_device *dev,
1331 bool new_carrier);
1332 int (*ndo_get_phys_port_id)(struct net_device *dev,
1333 struct netdev_phys_item_id *ppid);
1334 int (*ndo_get_phys_port_name)(struct net_device *dev,
1335 char *name, size_t len);
1336 void (*ndo_udp_tunnel_add)(struct net_device *dev,
1337 struct udp_tunnel_info *ti);
1338 void (*ndo_udp_tunnel_del)(struct net_device *dev,
1339 struct udp_tunnel_info *ti);
1340 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1341 struct net_device *dev);
1342 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1343 void *priv);
1344
1345 int (*ndo_get_lock_subclass)(struct net_device *dev);
1346 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1347 int queue_index,
1348 u32 maxrate);
1349 int (*ndo_get_iflink)(const struct net_device *dev);
1350 int (*ndo_change_proto_down)(struct net_device *dev,
1351 bool proto_down);
1352 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1353 struct sk_buff *skb);
1354 void (*ndo_set_rx_headroom)(struct net_device *dev,
1355 int needed_headroom);
1356 int (*ndo_bpf)(struct net_device *dev,
1357 struct netdev_bpf *bpf);
1358 int (*ndo_xdp_xmit)(struct net_device *dev,
1359 struct xdp_buff *xdp);
1360 void (*ndo_xdp_flush)(struct net_device *dev);
1361};
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405enum netdev_priv_flags {
1406 IFF_802_1Q_VLAN = 1<<0,
1407 IFF_EBRIDGE = 1<<1,
1408 IFF_BONDING = 1<<2,
1409 IFF_ISATAP = 1<<3,
1410 IFF_WAN_HDLC = 1<<4,
1411 IFF_XMIT_DST_RELEASE = 1<<5,
1412 IFF_DONT_BRIDGE = 1<<6,
1413 IFF_DISABLE_NETPOLL = 1<<7,
1414 IFF_MACVLAN_PORT = 1<<8,
1415 IFF_BRIDGE_PORT = 1<<9,
1416 IFF_OVS_DATAPATH = 1<<10,
1417 IFF_TX_SKB_SHARING = 1<<11,
1418 IFF_UNICAST_FLT = 1<<12,
1419 IFF_TEAM_PORT = 1<<13,
1420 IFF_SUPP_NOFCS = 1<<14,
1421 IFF_LIVE_ADDR_CHANGE = 1<<15,
1422 IFF_MACVLAN = 1<<16,
1423 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1424 IFF_L3MDEV_MASTER = 1<<18,
1425 IFF_NO_QUEUE = 1<<19,
1426 IFF_OPENVSWITCH = 1<<20,
1427 IFF_L3MDEV_SLAVE = 1<<21,
1428 IFF_TEAM = 1<<22,
1429 IFF_RXFH_CONFIGURED = 1<<23,
1430 IFF_PHONY_HEADROOM = 1<<24,
1431 IFF_MACSEC = 1<<25,
1432 IFF_NO_RX_HANDLER = 1<<26,
1433};
1434
1435#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
1436#define IFF_EBRIDGE IFF_EBRIDGE
1437#define IFF_BONDING IFF_BONDING
1438#define IFF_ISATAP IFF_ISATAP
1439#define IFF_WAN_HDLC IFF_WAN_HDLC
1440#define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
1441#define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
1442#define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
1443#define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
1444#define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
1445#define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
1446#define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
1447#define IFF_UNICAST_FLT IFF_UNICAST_FLT
1448#define IFF_TEAM_PORT IFF_TEAM_PORT
1449#define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
1450#define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
1451#define IFF_MACVLAN IFF_MACVLAN
1452#define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
1453#define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
1454#define IFF_NO_QUEUE IFF_NO_QUEUE
1455#define IFF_OPENVSWITCH IFF_OPENVSWITCH
1456#define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
1457#define IFF_TEAM IFF_TEAM
1458#define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED
1459#define IFF_MACSEC IFF_MACSEC
1460#define IFF_NO_RX_HANDLER IFF_NO_RX_HANDLER
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679struct net_device {
1680 char name[IFNAMSIZ];
1681 struct hlist_node name_hlist;
1682 struct dev_ifalias __rcu *ifalias;
1683
1684
1685
1686
1687 unsigned long mem_end;
1688 unsigned long mem_start;
1689 unsigned long base_addr;
1690 int irq;
1691
1692
1693
1694
1695
1696
1697
1698 unsigned long state;
1699
1700 struct list_head dev_list;
1701 struct list_head napi_list;
1702 struct list_head unreg_list;
1703 struct list_head close_list;
1704 struct list_head ptype_all;
1705 struct list_head ptype_specific;
1706
1707 struct {
1708 struct list_head upper;
1709 struct list_head lower;
1710 } adj_list;
1711
1712 netdev_features_t features;
1713 netdev_features_t hw_features;
1714 netdev_features_t wanted_features;
1715 netdev_features_t vlan_features;
1716 netdev_features_t hw_enc_features;
1717 netdev_features_t mpls_features;
1718 netdev_features_t gso_partial_features;
1719
1720 int ifindex;
1721 int group;
1722
1723 struct net_device_stats stats;
1724
1725 atomic_long_t rx_dropped;
1726 atomic_long_t tx_dropped;
1727 atomic_long_t rx_nohandler;
1728
1729
1730 atomic_t carrier_up_count;
1731 atomic_t carrier_down_count;
1732
1733#ifdef CONFIG_WIRELESS_EXT
1734 const struct iw_handler_def *wireless_handlers;
1735 struct iw_public_data *wireless_data;
1736#endif
1737 const struct net_device_ops *netdev_ops;
1738 const struct ethtool_ops *ethtool_ops;
1739#ifdef CONFIG_NET_SWITCHDEV
1740 const struct switchdev_ops *switchdev_ops;
1741#endif
1742#ifdef CONFIG_NET_L3_MASTER_DEV
1743 const struct l3mdev_ops *l3mdev_ops;
1744#endif
1745#if IS_ENABLED(CONFIG_IPV6)
1746 const struct ndisc_ops *ndisc_ops;
1747#endif
1748
1749#ifdef CONFIG_XFRM_OFFLOAD
1750 const struct xfrmdev_ops *xfrmdev_ops;
1751#endif
1752
1753 const struct header_ops *header_ops;
1754
1755 unsigned int flags;
1756 unsigned int priv_flags;
1757
1758 unsigned short gflags;
1759 unsigned short padded;
1760
1761 unsigned char operstate;
1762 unsigned char link_mode;
1763
1764 unsigned char if_port;
1765 unsigned char dma;
1766
1767 unsigned int mtu;
1768 unsigned int min_mtu;
1769 unsigned int max_mtu;
1770 unsigned short type;
1771 unsigned short hard_header_len;
1772 unsigned char min_header_len;
1773
1774 unsigned short needed_headroom;
1775 unsigned short needed_tailroom;
1776
1777
1778 unsigned char perm_addr[MAX_ADDR_LEN];
1779 unsigned char addr_assign_type;
1780 unsigned char addr_len;
1781 unsigned short neigh_priv_len;
1782 unsigned short dev_id;
1783 unsigned short dev_port;
1784 spinlock_t addr_list_lock;
1785 unsigned char name_assign_type;
1786 bool uc_promisc;
1787 struct netdev_hw_addr_list uc;
1788 struct netdev_hw_addr_list mc;
1789 struct netdev_hw_addr_list dev_addrs;
1790
1791#ifdef CONFIG_SYSFS
1792 struct kset *queues_kset;
1793#endif
1794 unsigned int promiscuity;
1795 unsigned int allmulti;
1796
1797
1798
1799
1800#if IS_ENABLED(CONFIG_VLAN_8021Q)
1801 struct vlan_info __rcu *vlan_info;
1802#endif
1803#if IS_ENABLED(CONFIG_NET_DSA)
1804 struct dsa_port *dsa_ptr;
1805#endif
1806#if IS_ENABLED(CONFIG_TIPC)
1807 struct tipc_bearer __rcu *tipc_ptr;
1808#endif
1809#if IS_ENABLED(CONFIG_IRDA) || IS_ENABLED(CONFIG_ATALK)
1810 void *atalk_ptr;
1811#endif
1812 struct in_device __rcu *ip_ptr;
1813#if IS_ENABLED(CONFIG_DECNET)
1814 struct dn_dev __rcu *dn_ptr;
1815#endif
1816 struct inet6_dev __rcu *ip6_ptr;
1817#if IS_ENABLED(CONFIG_AX25)
1818 void *ax25_ptr;
1819#endif
1820 struct wireless_dev *ieee80211_ptr;
1821 struct wpan_dev *ieee802154_ptr;
1822#if IS_ENABLED(CONFIG_MPLS_ROUTING)
1823 struct mpls_dev __rcu *mpls_ptr;
1824#endif
1825
1826
1827
1828
1829
1830 unsigned char *dev_addr;
1831
1832 struct netdev_rx_queue *_rx;
1833 unsigned int num_rx_queues;
1834 unsigned int real_num_rx_queues;
1835
1836 struct bpf_prog __rcu *xdp_prog;
1837 unsigned long gro_flush_timeout;
1838 rx_handler_func_t __rcu *rx_handler;
1839 void __rcu *rx_handler_data;
1840
1841#ifdef CONFIG_NET_CLS_ACT
1842 struct mini_Qdisc __rcu *miniq_ingress;
1843#endif
1844 struct netdev_queue __rcu *ingress_queue;
1845#ifdef CONFIG_NETFILTER_INGRESS
1846 struct nf_hook_entries __rcu *nf_hooks_ingress;
1847#endif
1848
1849 unsigned char broadcast[MAX_ADDR_LEN];
1850#ifdef CONFIG_RFS_ACCEL
1851 struct cpu_rmap *rx_cpu_rmap;
1852#endif
1853 struct hlist_node index_hlist;
1854
1855
1856
1857
1858 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
1859 unsigned int num_tx_queues;
1860 unsigned int real_num_tx_queues;
1861 struct Qdisc *qdisc;
1862#ifdef CONFIG_NET_SCHED
1863 DECLARE_HASHTABLE (qdisc_hash, 4);
1864#endif
1865 unsigned int tx_queue_len;
1866 spinlock_t tx_global_lock;
1867 int watchdog_timeo;
1868
1869#ifdef CONFIG_XPS
1870 struct xps_dev_maps __rcu *xps_maps;
1871#endif
1872#ifdef CONFIG_NET_CLS_ACT
1873 struct mini_Qdisc __rcu *miniq_egress;
1874#endif
1875
1876
1877 struct timer_list watchdog_timer;
1878
1879 int __percpu *pcpu_refcnt;
1880 struct list_head todo_list;
1881
1882 struct list_head link_watch_list;
1883
1884 enum { NETREG_UNINITIALIZED=0,
1885 NETREG_REGISTERED,
1886 NETREG_UNREGISTERING,
1887 NETREG_UNREGISTERED,
1888 NETREG_RELEASED,
1889 NETREG_DUMMY,
1890 } reg_state:8;
1891
1892 bool dismantle;
1893
1894 enum {
1895 RTNL_LINK_INITIALIZED,
1896 RTNL_LINK_INITIALIZING,
1897 } rtnl_link_state:16;
1898
1899 bool needs_free_netdev;
1900 void (*priv_destructor)(struct net_device *dev);
1901
1902#ifdef CONFIG_NETPOLL
1903 struct netpoll_info __rcu *npinfo;
1904#endif
1905
1906 possible_net_t nd_net;
1907
1908
1909 union {
1910 void *ml_priv;
1911 struct pcpu_lstats __percpu *lstats;
1912 struct pcpu_sw_netstats __percpu *tstats;
1913 struct pcpu_dstats __percpu *dstats;
1914 struct pcpu_vstats __percpu *vstats;
1915 };
1916
1917#if IS_ENABLED(CONFIG_GARP)
1918 struct garp_port __rcu *garp_port;
1919#endif
1920#if IS_ENABLED(CONFIG_MRP)
1921 struct mrp_port __rcu *mrp_port;
1922#endif
1923
1924 struct device dev;
1925 const struct attribute_group *sysfs_groups[4];
1926 const struct attribute_group *sysfs_rx_queue_group;
1927
1928 const struct rtnl_link_ops *rtnl_link_ops;
1929
1930
1931#define GSO_MAX_SIZE 65536
1932 unsigned int gso_max_size;
1933#define GSO_MAX_SEGS 65535
1934 u16 gso_max_segs;
1935
1936#ifdef CONFIG_DCB
1937 const struct dcbnl_rtnl_ops *dcbnl_ops;
1938#endif
1939 u8 num_tc;
1940 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1941 u8 prio_tc_map[TC_BITMASK + 1];
1942
1943#if IS_ENABLED(CONFIG_FCOE)
1944 unsigned int fcoe_ddp_xid;
1945#endif
1946#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
1947 struct netprio_map __rcu *priomap;
1948#endif
1949 struct phy_device *phydev;
1950 struct sfp_bus *sfp_bus;
1951 struct lock_class_key *qdisc_tx_busylock;
1952 struct lock_class_key *qdisc_running_key;
1953 bool proto_down;
1954};
1955#define to_net_dev(d) container_of(d, struct net_device, dev)
1956
1957static inline bool netif_elide_gro(const struct net_device *dev)
1958{
1959 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
1960 return true;
1961 return false;
1962}
1963
1964#define NETDEV_ALIGN 32
1965
1966static inline
1967int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1968{
1969 return dev->prio_tc_map[prio & TC_BITMASK];
1970}
1971
1972static inline
1973int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1974{
1975 if (tc >= dev->num_tc)
1976 return -EINVAL;
1977
1978 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
1979 return 0;
1980}
1981
1982int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
1983void netdev_reset_tc(struct net_device *dev);
1984int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
1985int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
1986
1987static inline
1988int netdev_get_num_tc(struct net_device *dev)
1989{
1990 return dev->num_tc;
1991}
1992
1993static inline
1994struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
1995 unsigned int index)
1996{
1997 return &dev->_tx[index];
1998}
1999
2000static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2001 const struct sk_buff *skb)
2002{
2003 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2004}
2005
2006static inline void netdev_for_each_tx_queue(struct net_device *dev,
2007 void (*f)(struct net_device *,
2008 struct netdev_queue *,
2009 void *),
2010 void *arg)
2011{
2012 unsigned int i;
2013
2014 for (i = 0; i < dev->num_tx_queues; i++)
2015 f(dev, &dev->_tx[i], arg);
2016}
2017
2018#define netdev_lockdep_set_classes(dev) \
2019{ \
2020 static struct lock_class_key qdisc_tx_busylock_key; \
2021 static struct lock_class_key qdisc_running_key; \
2022 static struct lock_class_key qdisc_xmit_lock_key; \
2023 static struct lock_class_key dev_addr_list_lock_key; \
2024 unsigned int i; \
2025 \
2026 (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \
2027 (dev)->qdisc_running_key = &qdisc_running_key; \
2028 lockdep_set_class(&(dev)->addr_list_lock, \
2029 &dev_addr_list_lock_key); \
2030 for (i = 0; i < (dev)->num_tx_queues; i++) \
2031 lockdep_set_class(&(dev)->_tx[i]._xmit_lock, \
2032 &qdisc_xmit_lock_key); \
2033}
2034
2035struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2036 struct sk_buff *skb,
2037 void *accel_priv);
2038
2039
2040
2041
2042static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2043{
2044 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2045}
2046
2047static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2048{
2049 if (dev->netdev_ops->ndo_set_rx_headroom)
2050 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2051}
2052
2053
2054static inline void netdev_reset_rx_headroom(struct net_device *dev)
2055{
2056 netdev_set_rx_headroom(dev, -1);
2057}
2058
2059
2060
2061
2062static inline
2063struct net *dev_net(const struct net_device *dev)
2064{
2065 return read_pnet(&dev->nd_net);
2066}
2067
2068static inline
2069void dev_net_set(struct net_device *dev, struct net *net)
2070{
2071 write_pnet(&dev->nd_net, net);
2072}
2073
2074
2075
2076
2077
2078
2079
2080static inline void *netdev_priv(const struct net_device *dev)
2081{
2082 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
2083}
2084
2085
2086
2087
2088#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
2089
2090
2091
2092
2093
2094#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2095
2096
2097
2098
2099#define NAPI_POLL_WEIGHT 64
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2112 int (*poll)(struct napi_struct *, int), int weight);
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125static inline void netif_tx_napi_add(struct net_device *dev,
2126 struct napi_struct *napi,
2127 int (*poll)(struct napi_struct *, int),
2128 int weight)
2129{
2130 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2131 netif_napi_add(dev, napi, poll, weight);
2132}
2133
2134
2135
2136
2137
2138
2139
2140void netif_napi_del(struct napi_struct *napi);
2141
2142struct napi_gro_cb {
2143
2144 void *frag0;
2145
2146
2147 unsigned int frag0_len;
2148
2149
2150 int data_offset;
2151
2152
2153 u16 flush;
2154
2155
2156 u16 flush_id;
2157
2158
2159 u16 count;
2160
2161
2162 u16 gro_remcsum_start;
2163
2164
2165 unsigned long age;
2166
2167
2168 u16 proto;
2169
2170
2171 u8 same_flow:1;
2172
2173
2174 u8 encap_mark:1;
2175
2176
2177 u8 csum_valid:1;
2178
2179
2180 u8 csum_cnt:3;
2181
2182
2183 u8 free:2;
2184#define NAPI_GRO_FREE 1
2185#define NAPI_GRO_FREE_STOLEN_HEAD 2
2186
2187
2188 u8 is_ipv6:1;
2189
2190
2191 u8 is_fou:1;
2192
2193
2194 u8 is_atomic:1;
2195
2196
2197 u8 recursion_counter:4;
2198
2199
2200
2201
2202 __wsum csum;
2203
2204
2205 struct sk_buff *last;
2206};
2207
2208#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
2209
2210#define GRO_RECURSION_LIMIT 15
2211static inline int gro_recursion_inc_test(struct sk_buff *skb)
2212{
2213 return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
2214}
2215
2216typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);
2217static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
2218 struct sk_buff **head,
2219 struct sk_buff *skb)
2220{
2221 if (unlikely(gro_recursion_inc_test(skb))) {
2222 NAPI_GRO_CB(skb)->flush |= 1;
2223 return NULL;
2224 }
2225
2226 return cb(head, skb);
2227}
2228
2229typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,
2230 struct sk_buff *);
2231static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,
2232 struct sock *sk,
2233 struct sk_buff **head,
2234 struct sk_buff *skb)
2235{
2236 if (unlikely(gro_recursion_inc_test(skb))) {
2237 NAPI_GRO_CB(skb)->flush |= 1;
2238 return NULL;
2239 }
2240
2241 return cb(sk, head, skb);
2242}
2243
2244struct packet_type {
2245 __be16 type;
2246 struct net_device *dev;
2247 int (*func) (struct sk_buff *,
2248 struct net_device *,
2249 struct packet_type *,
2250 struct net_device *);
2251 bool (*id_match)(struct packet_type *ptype,
2252 struct sock *sk);
2253 void *af_packet_priv;
2254 struct list_head list;
2255};
2256
2257struct offload_callbacks {
2258 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2259 netdev_features_t features);
2260 struct sk_buff **(*gro_receive)(struct sk_buff **head,
2261 struct sk_buff *skb);
2262 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2263};
2264
2265struct packet_offload {
2266 __be16 type;
2267 u16 priority;
2268 struct offload_callbacks callbacks;
2269 struct list_head list;
2270};
2271
2272
2273struct pcpu_sw_netstats {
2274 u64 rx_packets;
2275 u64 rx_bytes;
2276 u64 tx_packets;
2277 u64 tx_bytes;
2278 struct u64_stats_sync syncp;
2279};
2280
2281#define __netdev_alloc_pcpu_stats(type, gfp) \
2282({ \
2283 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2284 if (pcpu_stats) { \
2285 int __cpu; \
2286 for_each_possible_cpu(__cpu) { \
2287 typeof(type) *stat; \
2288 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2289 u64_stats_init(&stat->syncp); \
2290 } \
2291 } \
2292 pcpu_stats; \
2293})
2294
2295#define netdev_alloc_pcpu_stats(type) \
2296 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2297
2298enum netdev_lag_tx_type {
2299 NETDEV_LAG_TX_TYPE_UNKNOWN,
2300 NETDEV_LAG_TX_TYPE_RANDOM,
2301 NETDEV_LAG_TX_TYPE_BROADCAST,
2302 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2303 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2304 NETDEV_LAG_TX_TYPE_HASH,
2305};
2306
2307struct netdev_lag_upper_info {
2308 enum netdev_lag_tx_type tx_type;
2309};
2310
2311struct netdev_lag_lower_state_info {
2312 u8 link_up : 1,
2313 tx_enabled : 1;
2314};
2315
2316#include <linux/notifier.h>
2317
2318
2319
2320
2321
2322enum netdev_cmd {
2323 NETDEV_UP = 1,
2324 NETDEV_DOWN,
2325 NETDEV_REBOOT,
2326
2327
2328
2329 NETDEV_CHANGE,
2330 NETDEV_REGISTER,
2331 NETDEV_UNREGISTER,
2332 NETDEV_CHANGEMTU,
2333 NETDEV_CHANGEADDR,
2334 NETDEV_GOING_DOWN,
2335 NETDEV_CHANGENAME,
2336 NETDEV_FEAT_CHANGE,
2337 NETDEV_BONDING_FAILOVER,
2338 NETDEV_PRE_UP,
2339 NETDEV_PRE_TYPE_CHANGE,
2340 NETDEV_POST_TYPE_CHANGE,
2341 NETDEV_POST_INIT,
2342 NETDEV_RELEASE,
2343 NETDEV_NOTIFY_PEERS,
2344 NETDEV_JOIN,
2345 NETDEV_CHANGEUPPER,
2346 NETDEV_RESEND_IGMP,
2347 NETDEV_PRECHANGEMTU,
2348 NETDEV_CHANGEINFODATA,
2349 NETDEV_BONDING_INFO,
2350 NETDEV_PRECHANGEUPPER,
2351 NETDEV_CHANGELOWERSTATE,
2352 NETDEV_UDP_TUNNEL_PUSH_INFO,
2353 NETDEV_UDP_TUNNEL_DROP_INFO,
2354 NETDEV_CHANGE_TX_QUEUE_LEN,
2355 NETDEV_CVLAN_FILTER_PUSH_INFO,
2356 NETDEV_CVLAN_FILTER_DROP_INFO,
2357 NETDEV_SVLAN_FILTER_PUSH_INFO,
2358 NETDEV_SVLAN_FILTER_DROP_INFO,
2359};
2360const char *netdev_cmd_to_name(enum netdev_cmd cmd);
2361
2362int register_netdevice_notifier(struct notifier_block *nb);
2363int unregister_netdevice_notifier(struct notifier_block *nb);
2364
2365struct netdev_notifier_info {
2366 struct net_device *dev;
2367 struct netlink_ext_ack *extack;
2368};
2369
2370struct netdev_notifier_change_info {
2371 struct netdev_notifier_info info;
2372 unsigned int flags_changed;
2373};
2374
2375struct netdev_notifier_changeupper_info {
2376 struct netdev_notifier_info info;
2377 struct net_device *upper_dev;
2378 bool master;
2379 bool linking;
2380 void *upper_info;
2381};
2382
2383struct netdev_notifier_changelowerstate_info {
2384 struct netdev_notifier_info info;
2385 void *lower_state_info;
2386};
2387
2388static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2389 struct net_device *dev)
2390{
2391 info->dev = dev;
2392 info->extack = NULL;
2393}
2394
2395static inline struct net_device *
2396netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2397{
2398 return info->dev;
2399}
2400
2401static inline struct netlink_ext_ack *
2402netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2403{
2404 return info->extack;
2405}
2406
2407int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2408
2409
2410extern rwlock_t dev_base_lock;
2411
2412#define for_each_netdev(net, d) \
2413 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2414#define for_each_netdev_reverse(net, d) \
2415 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2416#define for_each_netdev_rcu(net, d) \
2417 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2418#define for_each_netdev_safe(net, d, n) \
2419 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2420#define for_each_netdev_continue(net, d) \
2421 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2422#define for_each_netdev_continue_rcu(net, d) \
2423 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2424#define for_each_netdev_in_bond_rcu(bond, slave) \
2425 for_each_netdev_rcu(&init_net, slave) \
2426 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2427#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
2428
2429static inline struct net_device *next_net_device(struct net_device *dev)
2430{
2431 struct list_head *lh;
2432 struct net *net;
2433
2434 net = dev_net(dev);
2435 lh = dev->dev_list.next;
2436 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2437}
2438
2439static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2440{
2441 struct list_head *lh;
2442 struct net *net;
2443
2444 net = dev_net(dev);
2445 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2446 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2447}
2448
2449static inline struct net_device *first_net_device(struct net *net)
2450{
2451 return list_empty(&net->dev_base_head) ? NULL :
2452 net_device_entry(net->dev_base_head.next);
2453}
2454
2455static inline struct net_device *first_net_device_rcu(struct net *net)
2456{
2457 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2458
2459 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2460}
2461
2462int netdev_boot_setup_check(struct net_device *dev);
2463unsigned long netdev_boot_base(const char *prefix, int unit);
2464struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2465 const char *hwaddr);
2466struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2467struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
2468void dev_add_pack(struct packet_type *pt);
2469void dev_remove_pack(struct packet_type *pt);
2470void __dev_remove_pack(struct packet_type *pt);
2471void dev_add_offload(struct packet_offload *po);
2472void dev_remove_offload(struct packet_offload *po);
2473
2474int dev_get_iflink(const struct net_device *dev);
2475int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
2476struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2477 unsigned short mask);
2478struct net_device *dev_get_by_name(struct net *net, const char *name);
2479struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2480struct net_device *__dev_get_by_name(struct net *net, const char *name);
2481int dev_alloc_name(struct net_device *dev, const char *name);
2482int dev_open(struct net_device *dev);
2483void dev_close(struct net_device *dev);
2484void dev_close_many(struct list_head *head, bool unlink);
2485void dev_disable_lro(struct net_device *dev);
2486int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
2487int dev_queue_xmit(struct sk_buff *skb);
2488int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
2489int register_netdevice(struct net_device *dev);
2490void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
2491void unregister_netdevice_many(struct list_head *head);
2492static inline void unregister_netdevice(struct net_device *dev)
2493{
2494 unregister_netdevice_queue(dev, NULL);
2495}
2496
2497int netdev_refcnt_read(const struct net_device *dev);
2498void free_netdev(struct net_device *dev);
2499void netdev_freemem(struct net_device *dev);
2500void synchronize_net(void);
2501int init_dummy_netdev(struct net_device *dev);
2502
2503DECLARE_PER_CPU(int, xmit_recursion);
2504#define XMIT_RECURSION_LIMIT 10
2505
2506static inline int dev_recursion_level(void)
2507{
2508 return this_cpu_read(xmit_recursion);
2509}
2510
2511struct net_device *dev_get_by_index(struct net *net, int ifindex);
2512struct net_device *__dev_get_by_index(struct net *net, int ifindex);
2513struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
2514struct net_device *dev_get_by_napi_id(unsigned int napi_id);
2515int netdev_get_name(struct net *net, char *name, int ifindex);
2516int dev_restart(struct net_device *dev);
2517int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2518
2519static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
2520{
2521 return NAPI_GRO_CB(skb)->data_offset;
2522}
2523
2524static inline unsigned int skb_gro_len(const struct sk_buff *skb)
2525{
2526 return skb->len - NAPI_GRO_CB(skb)->data_offset;
2527}
2528
2529static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
2530{
2531 NAPI_GRO_CB(skb)->data_offset += len;
2532}
2533
2534static inline void *skb_gro_header_fast(struct sk_buff *skb,
2535 unsigned int offset)
2536{
2537 return NAPI_GRO_CB(skb)->frag0 + offset;
2538}
2539
2540static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
2541{
2542 return NAPI_GRO_CB(skb)->frag0_len < hlen;
2543}
2544
2545static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
2546{
2547 NAPI_GRO_CB(skb)->frag0 = NULL;
2548 NAPI_GRO_CB(skb)->frag0_len = 0;
2549}
2550
2551static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
2552 unsigned int offset)
2553{
2554 if (!pskb_may_pull(skb, hlen))
2555 return NULL;
2556
2557 skb_gro_frag0_invalidate(skb);
2558 return skb->data + offset;
2559}
2560
2561static inline void *skb_gro_network_header(struct sk_buff *skb)
2562{
2563 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
2564 skb_network_offset(skb);
2565}
2566
2567static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
2568 const void *start, unsigned int len)
2569{
2570 if (NAPI_GRO_CB(skb)->csum_valid)
2571 NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
2572 csum_partial(start, len, 0));
2573}
2574
2575
2576
2577
2578
2579
2580__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
2581
2582static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
2583{
2584 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
2585}
2586
2587static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
2588 bool zero_okay,
2589 __sum16 check)
2590{
2591 return ((skb->ip_summed != CHECKSUM_PARTIAL ||
2592 skb_checksum_start_offset(skb) <
2593 skb_gro_offset(skb)) &&
2594 !skb_at_gro_remcsum_start(skb) &&
2595 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2596 (!zero_okay || check));
2597}
2598
2599static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
2600 __wsum psum)
2601{
2602 if (NAPI_GRO_CB(skb)->csum_valid &&
2603 !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
2604 return 0;
2605
2606 NAPI_GRO_CB(skb)->csum = psum;
2607
2608 return __skb_gro_checksum_complete(skb);
2609}
2610
2611static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
2612{
2613 if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
2614
2615 NAPI_GRO_CB(skb)->csum_cnt--;
2616 } else {
2617
2618
2619
2620
2621 __skb_incr_checksum_unnecessary(skb);
2622 }
2623}
2624
2625#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \
2626 compute_pseudo) \
2627({ \
2628 __sum16 __ret = 0; \
2629 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \
2630 __ret = __skb_gro_checksum_validate_complete(skb, \
2631 compute_pseudo(skb, proto)); \
2632 if (!__ret) \
2633 skb_gro_incr_csum_unnecessary(skb); \
2634 __ret; \
2635})
2636
2637#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \
2638 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
2639
2640#define skb_gro_checksum_validate_zero_check(skb, proto, check, \
2641 compute_pseudo) \
2642 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
2643
2644#define skb_gro_checksum_simple_validate(skb) \
2645 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
2646
2647static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
2648{
2649 return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2650 !NAPI_GRO_CB(skb)->csum_valid);
2651}
2652
2653static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2654 __sum16 check, __wsum pseudo)
2655{
2656 NAPI_GRO_CB(skb)->csum = ~pseudo;
2657 NAPI_GRO_CB(skb)->csum_valid = 1;
2658}
2659
2660#define skb_gro_checksum_try_convert(skb, proto, check, compute_pseudo) \
2661do { \
2662 if (__skb_gro_checksum_convert_check(skb)) \
2663 __skb_gro_checksum_convert(skb, check, \
2664 compute_pseudo(skb, proto)); \
2665} while (0)
2666
2667struct gro_remcsum {
2668 int offset;
2669 __wsum delta;
2670};
2671
2672static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
2673{
2674 grc->offset = 0;
2675 grc->delta = 0;
2676}
2677
2678static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
2679 unsigned int off, size_t hdrlen,
2680 int start, int offset,
2681 struct gro_remcsum *grc,
2682 bool nopartial)
2683{
2684 __wsum delta;
2685 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
2686
2687 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
2688
2689 if (!nopartial) {
2690 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
2691 return ptr;
2692 }
2693
2694 ptr = skb_gro_header_fast(skb, off);
2695 if (skb_gro_header_hard(skb, off + plen)) {
2696 ptr = skb_gro_header_slow(skb, off + plen, off);
2697 if (!ptr)
2698 return NULL;
2699 }
2700
2701 delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
2702 start, offset);
2703
2704
2705 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
2706
2707 grc->offset = off + hdrlen + offset;
2708 grc->delta = delta;
2709
2710 return ptr;
2711}
2712
2713static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
2714 struct gro_remcsum *grc)
2715{
2716 void *ptr;
2717 size_t plen = grc->offset + sizeof(u16);
2718
2719 if (!grc->delta)
2720 return;
2721
2722 ptr = skb_gro_header_fast(skb, grc->offset);
2723 if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
2724 ptr = skb_gro_header_slow(skb, plen, grc->offset);
2725 if (!ptr)
2726 return;
2727 }
2728
2729 remcsum_unadjust((__sum16 *)ptr, grc->delta);
2730}
2731
2732#ifdef CONFIG_XFRM_OFFLOAD
2733static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2734{
2735 if (PTR_ERR(pp) != -EINPROGRESS)
2736 NAPI_GRO_CB(skb)->flush |= flush;
2737}
2738#else
2739static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2740{
2741 NAPI_GRO_CB(skb)->flush |= flush;
2742}
2743#endif
2744
2745static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
2746 unsigned short type,
2747 const void *daddr, const void *saddr,
2748 unsigned int len)
2749{
2750 if (!dev->header_ops || !dev->header_ops->create)
2751 return 0;
2752
2753 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
2754}
2755
2756static inline int dev_parse_header(const struct sk_buff *skb,
2757 unsigned char *haddr)
2758{
2759 const struct net_device *dev = skb->dev;
2760
2761 if (!dev->header_ops || !dev->header_ops->parse)
2762 return 0;
2763 return dev->header_ops->parse(skb, haddr);
2764}
2765
2766
2767static inline bool dev_validate_header(const struct net_device *dev,
2768 char *ll_header, int len)
2769{
2770 if (likely(len >= dev->hard_header_len))
2771 return true;
2772 if (len < dev->min_header_len)
2773 return false;
2774
2775 if (capable(CAP_SYS_RAWIO)) {
2776 memset(ll_header + len, 0, dev->hard_header_len - len);
2777 return true;
2778 }
2779
2780 if (dev->header_ops && dev->header_ops->validate)
2781 return dev->header_ops->validate(ll_header, len);
2782
2783 return false;
2784}
2785
2786typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr,
2787 int len, int size);
2788int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
2789static inline int unregister_gifconf(unsigned int family)
2790{
2791 return register_gifconf(family, NULL);
2792}
2793
2794#ifdef CONFIG_NET_FLOW_LIMIT
2795#define FLOW_LIMIT_HISTORY (1 << 7)
2796struct sd_flow_limit {
2797 u64 count;
2798 unsigned int num_buckets;
2799 unsigned int history_head;
2800 u16 history[FLOW_LIMIT_HISTORY];
2801 u8 buckets[];
2802};
2803
2804extern int netdev_flow_limit_table_len;
2805#endif
2806
2807
2808
2809
2810struct softnet_data {
2811 struct list_head poll_list;
2812 struct sk_buff_head process_queue;
2813
2814
2815 unsigned int processed;
2816 unsigned int time_squeeze;
2817 unsigned int received_rps;
2818#ifdef CONFIG_RPS
2819 struct softnet_data *rps_ipi_list;
2820#endif
2821#ifdef CONFIG_NET_FLOW_LIMIT
2822 struct sd_flow_limit __rcu *flow_limit;
2823#endif
2824 struct Qdisc *output_queue;
2825 struct Qdisc **output_queue_tailp;
2826 struct sk_buff *completion_queue;
2827#ifdef CONFIG_XFRM_OFFLOAD
2828 struct sk_buff_head xfrm_backlog;
2829#endif
2830#ifdef CONFIG_RPS
2831
2832
2833
2834 unsigned int input_queue_head ____cacheline_aligned_in_smp;
2835
2836
2837 call_single_data_t csd ____cacheline_aligned_in_smp;
2838 struct softnet_data *rps_ipi_next;
2839 unsigned int cpu;
2840 unsigned int input_queue_tail;
2841#endif
2842 unsigned int dropped;
2843 struct sk_buff_head input_pkt_queue;
2844 struct napi_struct backlog;
2845
2846};
2847
2848static inline void input_queue_head_incr(struct softnet_data *sd)
2849{
2850#ifdef CONFIG_RPS
2851 sd->input_queue_head++;
2852#endif
2853}
2854
2855static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2856 unsigned int *qtail)
2857{
2858#ifdef CONFIG_RPS
2859 *qtail = ++sd->input_queue_tail;
2860#endif
2861}
2862
2863DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
2864
2865void __netif_schedule(struct Qdisc *q);
2866void netif_schedule_queue(struct netdev_queue *txq);
2867
2868static inline void netif_tx_schedule_all(struct net_device *dev)
2869{
2870 unsigned int i;
2871
2872 for (i = 0; i < dev->num_tx_queues; i++)
2873 netif_schedule_queue(netdev_get_tx_queue(dev, i));
2874}
2875
2876static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
2877{
2878 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2879}
2880
2881
2882
2883
2884
2885
2886
2887static inline void netif_start_queue(struct net_device *dev)
2888{
2889 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
2890}
2891
2892static inline void netif_tx_start_all_queues(struct net_device *dev)
2893{
2894 unsigned int i;
2895
2896 for (i = 0; i < dev->num_tx_queues; i++) {
2897 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2898 netif_tx_start_queue(txq);
2899 }
2900}
2901
2902void netif_tx_wake_queue(struct netdev_queue *dev_queue);
2903
2904
2905
2906
2907
2908
2909
2910
2911static inline void netif_wake_queue(struct net_device *dev)
2912{
2913 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
2914}
2915
2916static inline void netif_tx_wake_all_queues(struct net_device *dev)
2917{
2918 unsigned int i;
2919
2920 for (i = 0; i < dev->num_tx_queues; i++) {
2921 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2922 netif_tx_wake_queue(txq);
2923 }
2924}
2925
2926static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
2927{
2928 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2929}
2930
2931
2932
2933
2934
2935
2936
2937
2938static inline void netif_stop_queue(struct net_device *dev)
2939{
2940 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
2941}
2942
2943void netif_tx_stop_all_queues(struct net_device *dev);
2944
2945static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
2946{
2947 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2948}
2949
2950
2951
2952
2953
2954
2955
2956static inline bool netif_queue_stopped(const struct net_device *dev)
2957{
2958 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
2959}
2960
2961static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
2962{
2963 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2964}
2965
2966static inline bool
2967netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
2968{
2969 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2970}
2971
2972static inline bool
2973netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
2974{
2975 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
2976}
2977
2978
2979
2980
2981
2982
2983
2984
2985static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
2986{
2987#ifdef CONFIG_BQL
2988 prefetchw(&dev_queue->dql.num_queued);
2989#endif
2990}
2991
2992
2993
2994
2995
2996
2997
2998
2999static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3000{
3001#ifdef CONFIG_BQL
3002 prefetchw(&dev_queue->dql.limit);
3003#endif
3004}
3005
3006static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3007 unsigned int bytes)
3008{
3009#ifdef CONFIG_BQL
3010 dql_queued(&dev_queue->dql, bytes);
3011
3012 if (likely(dql_avail(&dev_queue->dql) >= 0))
3013 return;
3014
3015 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3016
3017
3018
3019
3020
3021
3022 smp_mb();
3023
3024
3025 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3026 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3027#endif
3028}
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3040{
3041 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3042}
3043
3044static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3045 unsigned int pkts, unsigned int bytes)
3046{
3047#ifdef CONFIG_BQL
3048 if (unlikely(!bytes))
3049 return;
3050
3051 dql_completed(&dev_queue->dql, bytes);
3052
3053
3054
3055
3056
3057
3058 smp_mb();
3059
3060 if (dql_avail(&dev_queue->dql) < 0)
3061 return;
3062
3063 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3064 netif_schedule_queue(dev_queue);
3065#endif
3066}
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078static inline void netdev_completed_queue(struct net_device *dev,
3079 unsigned int pkts, unsigned int bytes)
3080{
3081 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3082}
3083
3084static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3085{
3086#ifdef CONFIG_BQL
3087 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3088 dql_reset(&q->dql);
3089#endif
3090}
3091
3092
3093
3094
3095
3096
3097
3098
3099static inline void netdev_reset_queue(struct net_device *dev_queue)
3100{
3101 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
3102}
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3113{
3114 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3115 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3116 dev->name, queue_index,
3117 dev->real_num_tx_queues);
3118 return 0;
3119 }
3120
3121 return queue_index;
3122}
3123
3124
3125
3126
3127
3128
3129
3130static inline bool netif_running(const struct net_device *dev)
3131{
3132 return test_bit(__LINK_STATE_START, &dev->state);
3133}
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3150{
3151 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3152
3153 netif_tx_start_queue(txq);
3154}
3155
3156
3157
3158
3159
3160
3161
3162
3163static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3164{
3165 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3166 netif_tx_stop_queue(txq);
3167}
3168
3169
3170
3171
3172
3173
3174
3175
3176static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3177 u16 queue_index)
3178{
3179 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3180
3181 return netif_tx_queue_stopped(txq);
3182}
3183
3184static inline bool netif_subqueue_stopped(const struct net_device *dev,
3185 struct sk_buff *skb)
3186{
3187 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3188}
3189
3190
3191
3192
3193
3194
3195
3196
3197static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3198{
3199 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3200
3201 netif_tx_wake_queue(txq);
3202}
3203
3204#ifdef CONFIG_XPS
3205int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3206 u16 index);
3207#else
3208static inline int netif_set_xps_queue(struct net_device *dev,
3209 const struct cpumask *mask,
3210 u16 index)
3211{
3212 return 0;
3213}
3214#endif
3215
3216u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
3217 unsigned int num_tx_queues);
3218
3219
3220
3221
3222
3223static inline u16 skb_tx_hash(const struct net_device *dev,
3224 struct sk_buff *skb)
3225{
3226 return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
3227}
3228
3229
3230
3231
3232
3233
3234
3235static inline bool netif_is_multiqueue(const struct net_device *dev)
3236{
3237 return dev->num_tx_queues > 1;
3238}
3239
3240int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3241
3242#ifdef CONFIG_SYSFS
3243int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3244#else
3245static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3246 unsigned int rxq)
3247{
3248 return 0;
3249}
3250#endif
3251
3252static inline struct netdev_rx_queue *
3253__netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
3254{
3255 return dev->_rx + rxq;
3256}
3257
3258#ifdef CONFIG_SYSFS
3259static inline unsigned int get_netdev_rx_queue_index(
3260 struct netdev_rx_queue *queue)
3261{
3262 struct net_device *dev = queue->dev;
3263 int index = queue - dev->_rx;
3264
3265 BUG_ON(index >= dev->num_rx_queues);
3266 return index;
3267}
3268#endif
3269
3270#define DEFAULT_MAX_NUM_RSS_QUEUES (8)
3271int netif_get_num_default_rss_queues(void);
3272
3273enum skb_free_reason {
3274 SKB_REASON_CONSUMED,
3275 SKB_REASON_DROPPED,
3276};
3277
3278void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
3279void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3301{
3302 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
3303}
3304
3305static inline void dev_consume_skb_irq(struct sk_buff *skb)
3306{
3307 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
3308}
3309
3310static inline void dev_kfree_skb_any(struct sk_buff *skb)
3311{
3312 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
3313}
3314
3315static inline void dev_consume_skb_any(struct sk_buff *skb)
3316{
3317 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3318}
3319
3320void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3321int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
3322int netif_rx(struct sk_buff *skb);
3323int netif_rx_ni(struct sk_buff *skb);
3324int netif_receive_skb(struct sk_buff *skb);
3325int netif_receive_skb_core(struct sk_buff *skb);
3326gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3327void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3328struct sk_buff *napi_get_frags(struct napi_struct *napi);
3329gro_result_t napi_gro_frags(struct napi_struct *napi);
3330struct packet_offload *gro_find_receive_by_type(__be16 type);
3331struct packet_offload *gro_find_complete_by_type(__be16 type);
3332
3333static inline void napi_free_frags(struct napi_struct *napi)
3334{
3335 kfree_skb(napi->skb);
3336 napi->skb = NULL;
3337}
3338
3339bool netdev_is_rx_handler_busy(struct net_device *dev);
3340int netdev_rx_handler_register(struct net_device *dev,
3341 rx_handler_func_t *rx_handler,
3342 void *rx_handler_data);
3343void netdev_rx_handler_unregister(struct net_device *dev);
3344
3345bool dev_valid_name(const char *name);
3346int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3347 bool *need_copyout);
3348int dev_ifconf(struct net *net, struct ifconf *, int);
3349int dev_ethtool(struct net *net, struct ifreq *);
3350unsigned int dev_get_flags(const struct net_device *);
3351int __dev_change_flags(struct net_device *, unsigned int flags);
3352int dev_change_flags(struct net_device *, unsigned int);
3353void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3354 unsigned int gchanges);
3355int dev_change_name(struct net_device *, const char *);
3356int dev_set_alias(struct net_device *, const char *, size_t);
3357int dev_get_alias(const struct net_device *, char *, size_t);
3358int dev_change_net_namespace(struct net_device *, struct net *, const char *);
3359int __dev_set_mtu(struct net_device *, int);
3360int dev_set_mtu(struct net_device *, int);
3361int dev_change_tx_queue_len(struct net_device *, unsigned long);
3362void dev_set_group(struct net_device *, int);
3363int dev_set_mac_address(struct net_device *, struct sockaddr *);
3364int dev_change_carrier(struct net_device *, bool new_carrier);
3365int dev_get_phys_port_id(struct net_device *dev,
3366 struct netdev_phys_item_id *ppid);
3367int dev_get_phys_port_name(struct net_device *dev,
3368 char *name, size_t len);
3369int dev_change_proto_down(struct net_device *dev, bool proto_down);
3370struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3371struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3372 struct netdev_queue *txq, int *ret);
3373
3374typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
3375int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
3376 int fd, u32 flags);
3377void __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,
3378 struct netdev_bpf *xdp);
3379
3380int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3381int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3382bool is_skb_forwardable(const struct net_device *dev,
3383 const struct sk_buff *skb);
3384
3385static __always_inline int ____dev_forward_skb(struct net_device *dev,
3386 struct sk_buff *skb)
3387{
3388 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3389 unlikely(!is_skb_forwardable(dev, skb))) {
3390 atomic_long_inc(&dev->rx_dropped);
3391 kfree_skb(skb);
3392 return NET_RX_DROP;
3393 }
3394
3395 skb_scrub_packet(skb, true);
3396 skb->priority = 0;
3397 return 0;
3398}
3399
3400void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
3401
3402extern int netdev_budget;
3403extern unsigned int netdev_budget_usecs;
3404
3405
3406void netdev_run_todo(void);
3407
3408
3409
3410
3411
3412
3413
3414static inline void dev_put(struct net_device *dev)
3415{
3416 this_cpu_dec(*dev->pcpu_refcnt);
3417}
3418
3419
3420
3421
3422
3423
3424
3425static inline void dev_hold(struct net_device *dev)
3426{
3427 this_cpu_inc(*dev->pcpu_refcnt);
3428}
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439void linkwatch_init_dev(struct net_device *dev);
3440void linkwatch_fire_event(struct net_device *dev);
3441void linkwatch_forget_dev(struct net_device *dev);
3442
3443
3444
3445
3446
3447
3448
3449static inline bool netif_carrier_ok(const struct net_device *dev)
3450{
3451 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
3452}
3453
3454unsigned long dev_trans_start(struct net_device *dev);
3455
3456void __netdev_watchdog_up(struct net_device *dev);
3457
3458void netif_carrier_on(struct net_device *dev);
3459
3460void netif_carrier_off(struct net_device *dev);
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474static inline void netif_dormant_on(struct net_device *dev)
3475{
3476 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
3477 linkwatch_fire_event(dev);
3478}
3479
3480
3481
3482
3483
3484
3485
3486static inline void netif_dormant_off(struct net_device *dev)
3487{
3488 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
3489 linkwatch_fire_event(dev);
3490}
3491
3492
3493
3494
3495
3496
3497
3498static inline bool netif_dormant(const struct net_device *dev)
3499{
3500 return test_bit(__LINK_STATE_DORMANT, &dev->state);
3501}
3502
3503
3504
3505
3506
3507
3508
3509
3510static inline bool netif_oper_up(const struct net_device *dev)
3511{
3512 return (dev->operstate == IF_OPER_UP ||
3513 dev->operstate == IF_OPER_UNKNOWN );
3514}
3515
3516
3517
3518
3519
3520
3521
3522static inline bool netif_device_present(struct net_device *dev)
3523{
3524 return test_bit(__LINK_STATE_PRESENT, &dev->state);
3525}
3526
3527void netif_device_detach(struct net_device *dev);
3528
3529void netif_device_attach(struct net_device *dev);
3530
3531
3532
3533
3534
3535enum {
3536 NETIF_MSG_DRV = 0x0001,
3537 NETIF_MSG_PROBE = 0x0002,
3538 NETIF_MSG_LINK = 0x0004,
3539 NETIF_MSG_TIMER = 0x0008,
3540 NETIF_MSG_IFDOWN = 0x0010,
3541 NETIF_MSG_IFUP = 0x0020,
3542 NETIF_MSG_RX_ERR = 0x0040,
3543 NETIF_MSG_TX_ERR = 0x0080,
3544 NETIF_MSG_TX_QUEUED = 0x0100,
3545 NETIF_MSG_INTR = 0x0200,
3546 NETIF_MSG_TX_DONE = 0x0400,
3547 NETIF_MSG_RX_STATUS = 0x0800,
3548 NETIF_MSG_PKTDATA = 0x1000,
3549 NETIF_MSG_HW = 0x2000,
3550 NETIF_MSG_WOL = 0x4000,
3551};
3552
3553#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
3554#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
3555#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
3556#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
3557#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
3558#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
3559#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
3560#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
3561#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
3562#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
3563#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
3564#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
3565#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
3566#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
3567#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
3568
3569static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
3570{
3571
3572 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
3573 return default_msg_enable_bits;
3574 if (debug_value == 0)
3575 return 0;
3576
3577 return (1 << debug_value) - 1;
3578}
3579
3580static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
3581{
3582 spin_lock(&txq->_xmit_lock);
3583 txq->xmit_lock_owner = cpu;
3584}
3585
3586static inline bool __netif_tx_acquire(struct netdev_queue *txq)
3587{
3588 __acquire(&txq->_xmit_lock);
3589 return true;
3590}
3591
3592static inline void __netif_tx_release(struct netdev_queue *txq)
3593{
3594 __release(&txq->_xmit_lock);
3595}
3596
3597static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
3598{
3599 spin_lock_bh(&txq->_xmit_lock);
3600 txq->xmit_lock_owner = smp_processor_id();
3601}
3602
3603static inline bool __netif_tx_trylock(struct netdev_queue *txq)
3604{
3605 bool ok = spin_trylock(&txq->_xmit_lock);
3606 if (likely(ok))
3607 txq->xmit_lock_owner = smp_processor_id();
3608 return ok;
3609}
3610
3611static inline void __netif_tx_unlock(struct netdev_queue *txq)
3612{
3613 txq->xmit_lock_owner = -1;
3614 spin_unlock(&txq->_xmit_lock);
3615}
3616
3617static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
3618{
3619 txq->xmit_lock_owner = -1;
3620 spin_unlock_bh(&txq->_xmit_lock);
3621}
3622
3623static inline void txq_trans_update(struct netdev_queue *txq)
3624{
3625 if (txq->xmit_lock_owner != -1)
3626 txq->trans_start = jiffies;
3627}
3628
3629
3630static inline void netif_trans_update(struct net_device *dev)
3631{
3632 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
3633
3634 if (txq->trans_start != jiffies)
3635 txq->trans_start = jiffies;
3636}
3637
3638
3639
3640
3641
3642
3643
3644static inline void netif_tx_lock(struct net_device *dev)
3645{
3646 unsigned int i;
3647 int cpu;
3648
3649 spin_lock(&dev->tx_global_lock);
3650 cpu = smp_processor_id();
3651 for (i = 0; i < dev->num_tx_queues; i++) {
3652 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3653
3654
3655
3656
3657
3658
3659
3660 __netif_tx_lock(txq, cpu);
3661 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
3662 __netif_tx_unlock(txq);
3663 }
3664}
3665
3666static inline void netif_tx_lock_bh(struct net_device *dev)
3667{
3668 local_bh_disable();
3669 netif_tx_lock(dev);
3670}
3671
3672static inline void netif_tx_unlock(struct net_device *dev)
3673{
3674 unsigned int i;
3675
3676 for (i = 0; i < dev->num_tx_queues; i++) {
3677 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3678
3679
3680
3681
3682
3683 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
3684 netif_schedule_queue(txq);
3685 }
3686 spin_unlock(&dev->tx_global_lock);
3687}
3688
3689static inline void netif_tx_unlock_bh(struct net_device *dev)
3690{
3691 netif_tx_unlock(dev);
3692 local_bh_enable();
3693}
3694
3695#define HARD_TX_LOCK(dev, txq, cpu) { \
3696 if ((dev->features & NETIF_F_LLTX) == 0) { \
3697 __netif_tx_lock(txq, cpu); \
3698 } else { \
3699 __netif_tx_acquire(txq); \
3700 } \
3701}
3702
3703#define HARD_TX_TRYLOCK(dev, txq) \
3704 (((dev->features & NETIF_F_LLTX) == 0) ? \
3705 __netif_tx_trylock(txq) : \
3706 __netif_tx_acquire(txq))
3707
3708#define HARD_TX_UNLOCK(dev, txq) { \
3709 if ((dev->features & NETIF_F_LLTX) == 0) { \
3710 __netif_tx_unlock(txq); \
3711 } else { \
3712 __netif_tx_release(txq); \
3713 } \
3714}
3715
3716static inline void netif_tx_disable(struct net_device *dev)
3717{
3718 unsigned int i;
3719 int cpu;
3720
3721 local_bh_disable();
3722 cpu = smp_processor_id();
3723 for (i = 0; i < dev->num_tx_queues; i++) {
3724 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3725
3726 __netif_tx_lock(txq, cpu);
3727 netif_tx_stop_queue(txq);
3728 __netif_tx_unlock(txq);
3729 }
3730 local_bh_enable();
3731}
3732
3733static inline void netif_addr_lock(struct net_device *dev)
3734{
3735 spin_lock(&dev->addr_list_lock);
3736}
3737
3738static inline void netif_addr_lock_nested(struct net_device *dev)
3739{
3740 int subclass = SINGLE_DEPTH_NESTING;
3741
3742 if (dev->netdev_ops->ndo_get_lock_subclass)
3743 subclass = dev->netdev_ops->ndo_get_lock_subclass(dev);
3744
3745 spin_lock_nested(&dev->addr_list_lock, subclass);
3746}
3747
3748static inline void netif_addr_lock_bh(struct net_device *dev)
3749{
3750 spin_lock_bh(&dev->addr_list_lock);
3751}
3752
3753static inline void netif_addr_unlock(struct net_device *dev)
3754{
3755 spin_unlock(&dev->addr_list_lock);
3756}
3757
3758static inline void netif_addr_unlock_bh(struct net_device *dev)
3759{
3760 spin_unlock_bh(&dev->addr_list_lock);
3761}
3762
3763
3764
3765
3766
3767#define for_each_dev_addr(dev, ha) \
3768 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
3769
3770
3771
3772void ether_setup(struct net_device *dev);
3773
3774
3775struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
3776 unsigned char name_assign_type,
3777 void (*setup)(struct net_device *),
3778 unsigned int txqs, unsigned int rxqs);
3779int dev_get_valid_name(struct net *net, struct net_device *dev,
3780 const char *name);
3781
3782#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
3783 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
3784
3785#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
3786 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
3787 count)
3788
3789int register_netdev(struct net_device *dev);
3790void unregister_netdev(struct net_device *dev);
3791
3792
3793int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
3794 struct netdev_hw_addr_list *from_list, int addr_len);
3795void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
3796 struct netdev_hw_addr_list *from_list, int addr_len);
3797int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
3798 struct net_device *dev,
3799 int (*sync)(struct net_device *, const unsigned char *),
3800 int (*unsync)(struct net_device *,
3801 const unsigned char *));
3802void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
3803 struct net_device *dev,
3804 int (*unsync)(struct net_device *,
3805 const unsigned char *));
3806void __hw_addr_init(struct netdev_hw_addr_list *list);
3807
3808
3809int dev_addr_add(struct net_device *dev, const unsigned char *addr,
3810 unsigned char addr_type);
3811int dev_addr_del(struct net_device *dev, const unsigned char *addr,
3812 unsigned char addr_type);
3813void dev_addr_flush(struct net_device *dev);
3814int dev_addr_init(struct net_device *dev);
3815
3816
3817int dev_uc_add(struct net_device *dev, const unsigned char *addr);
3818int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
3819int dev_uc_del(struct net_device *dev, const unsigned char *addr);
3820int dev_uc_sync(struct net_device *to, struct net_device *from);
3821int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
3822void dev_uc_unsync(struct net_device *to, struct net_device *from);
3823void dev_uc_flush(struct net_device *dev);
3824void dev_uc_init(struct net_device *dev);
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835static inline int __dev_uc_sync(struct net_device *dev,
3836 int (*sync)(struct net_device *,
3837 const unsigned char *),
3838 int (*unsync)(struct net_device *,
3839 const unsigned char *))
3840{
3841 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
3842}
3843
3844
3845
3846
3847
3848
3849
3850
3851static inline void __dev_uc_unsync(struct net_device *dev,
3852 int (*unsync)(struct net_device *,
3853 const unsigned char *))
3854{
3855 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
3856}
3857
3858
3859int dev_mc_add(struct net_device *dev, const unsigned char *addr);
3860int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
3861int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
3862int dev_mc_del(struct net_device *dev, const unsigned char *addr);
3863int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
3864int dev_mc_sync(struct net_device *to, struct net_device *from);
3865int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
3866void dev_mc_unsync(struct net_device *to, struct net_device *from);
3867void dev_mc_flush(struct net_device *dev);
3868void dev_mc_init(struct net_device *dev);
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879static inline int __dev_mc_sync(struct net_device *dev,
3880 int (*sync)(struct net_device *,
3881 const unsigned char *),
3882 int (*unsync)(struct net_device *,
3883 const unsigned char *))
3884{
3885 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
3886}
3887
3888
3889
3890
3891
3892
3893
3894
3895static inline void __dev_mc_unsync(struct net_device *dev,
3896 int (*unsync)(struct net_device *,
3897 const unsigned char *))
3898{
3899 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
3900}
3901
3902
3903void dev_set_rx_mode(struct net_device *dev);
3904void __dev_set_rx_mode(struct net_device *dev);
3905int dev_set_promiscuity(struct net_device *dev, int inc);
3906int dev_set_allmulti(struct net_device *dev, int inc);
3907void netdev_state_change(struct net_device *dev);
3908void netdev_notify_peers(struct net_device *dev);
3909void netdev_features_change(struct net_device *dev);
3910
3911void dev_load(struct net *net, const char *name);
3912struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
3913 struct rtnl_link_stats64 *storage);
3914void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
3915 const struct net_device_stats *netdev_stats);
3916
3917extern int netdev_max_backlog;
3918extern int netdev_tstamp_prequeue;
3919extern int weight_p;
3920extern int dev_weight_rx_bias;
3921extern int dev_weight_tx_bias;
3922extern int dev_rx_weight;
3923extern int dev_tx_weight;
3924
3925bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
3926struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
3927 struct list_head **iter);
3928struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
3929 struct list_head **iter);
3930
3931
3932#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
3933 for (iter = &(dev)->adj_list.upper, \
3934 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
3935 updev; \
3936 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
3937
3938int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
3939 int (*fn)(struct net_device *upper_dev,
3940 void *data),
3941 void *data);
3942
3943bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
3944 struct net_device *upper_dev);
3945
3946bool netdev_has_any_upper_dev(struct net_device *dev);
3947
3948void *netdev_lower_get_next_private(struct net_device *dev,
3949 struct list_head **iter);
3950void *netdev_lower_get_next_private_rcu(struct net_device *dev,
3951 struct list_head **iter);
3952
3953#define netdev_for_each_lower_private(dev, priv, iter) \
3954 for (iter = (dev)->adj_list.lower.next, \
3955 priv = netdev_lower_get_next_private(dev, &(iter)); \
3956 priv; \
3957 priv = netdev_lower_get_next_private(dev, &(iter)))
3958
3959#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
3960 for (iter = &(dev)->adj_list.lower, \
3961 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
3962 priv; \
3963 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
3964
3965void *netdev_lower_get_next(struct net_device *dev,
3966 struct list_head **iter);
3967
3968#define netdev_for_each_lower_dev(dev, ldev, iter) \
3969 for (iter = (dev)->adj_list.lower.next, \
3970 ldev = netdev_lower_get_next(dev, &(iter)); \
3971 ldev; \
3972 ldev = netdev_lower_get_next(dev, &(iter)))
3973
3974struct net_device *netdev_all_lower_get_next(struct net_device *dev,
3975 struct list_head **iter);
3976struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev,
3977 struct list_head **iter);
3978
3979int netdev_walk_all_lower_dev(struct net_device *dev,
3980 int (*fn)(struct net_device *lower_dev,
3981 void *data),
3982 void *data);
3983int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
3984 int (*fn)(struct net_device *lower_dev,
3985 void *data),
3986 void *data);
3987
3988void *netdev_adjacent_get_private(struct list_head *adj_list);
3989void *netdev_lower_get_first_private_rcu(struct net_device *dev);
3990struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
3991struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
3992int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
3993 struct netlink_ext_ack *extack);
3994int netdev_master_upper_dev_link(struct net_device *dev,
3995 struct net_device *upper_dev,
3996 void *upper_priv, void *upper_info,
3997 struct netlink_ext_ack *extack);
3998void netdev_upper_dev_unlink(struct net_device *dev,
3999 struct net_device *upper_dev);
4000void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
4001void *netdev_lower_dev_get_private(struct net_device *dev,
4002 struct net_device *lower_dev);
4003void netdev_lower_state_changed(struct net_device *lower_dev,
4004 void *lower_state_info);
4005
4006
4007#define NETDEV_RSS_KEY_LEN 52
4008extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
4009void netdev_rss_key_fill(void *buffer, size_t len);
4010
4011int dev_get_nest_level(struct net_device *dev);
4012int skb_checksum_help(struct sk_buff *skb);
4013int skb_crc32c_csum_help(struct sk_buff *skb);
4014int skb_csum_hwoffload_help(struct sk_buff *skb,
4015 const netdev_features_t features);
4016
4017struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
4018 netdev_features_t features, bool tx_path);
4019struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
4020 netdev_features_t features);
4021
4022struct netdev_bonding_info {
4023 ifslave slave;
4024 ifbond master;
4025};
4026
4027struct netdev_notifier_bonding_info {
4028 struct netdev_notifier_info info;
4029 struct netdev_bonding_info bonding_info;
4030};
4031
4032void netdev_bonding_info_change(struct net_device *dev,
4033 struct netdev_bonding_info *bonding_info);
4034
4035static inline
4036struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
4037{
4038 return __skb_gso_segment(skb, features, true);
4039}
4040__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4041
4042static inline bool can_checksum_protocol(netdev_features_t features,
4043 __be16 protocol)
4044{
4045 if (protocol == htons(ETH_P_FCOE))
4046 return !!(features & NETIF_F_FCOE_CRC);
4047
4048
4049
4050 if (features & NETIF_F_HW_CSUM) {
4051
4052 return true;
4053 }
4054
4055 switch (protocol) {
4056 case htons(ETH_P_IP):
4057 return !!(features & NETIF_F_IP_CSUM);
4058 case htons(ETH_P_IPV6):
4059 return !!(features & NETIF_F_IPV6_CSUM);
4060 default:
4061 return false;
4062 }
4063}
4064
4065#ifdef CONFIG_BUG
4066void netdev_rx_csum_fault(struct net_device *dev);
4067#else
4068static inline void netdev_rx_csum_fault(struct net_device *dev)
4069{
4070}
4071#endif
4072
4073void net_enable_timestamp(void);
4074void net_disable_timestamp(void);
4075
4076#ifdef CONFIG_PROC_FS
4077int __init dev_proc_init(void);
4078#else
4079#define dev_proc_init() 0
4080#endif
4081
4082static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4083 struct sk_buff *skb, struct net_device *dev,
4084 bool more)
4085{
4086 skb->xmit_more = more ? 1 : 0;
4087 return ops->ndo_start_xmit(skb, dev);
4088}
4089
4090static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4091 struct netdev_queue *txq, bool more)
4092{
4093 const struct net_device_ops *ops = dev->netdev_ops;
4094 int rc;
4095
4096 rc = __netdev_start_xmit(ops, skb, dev, more);
4097 if (rc == NETDEV_TX_OK)
4098 txq_trans_update(txq);
4099
4100 return rc;
4101}
4102
4103int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4104 const void *ns);
4105void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4106 const void *ns);
4107
4108static inline int netdev_class_create_file(const struct class_attribute *class_attr)
4109{
4110 return netdev_class_create_file_ns(class_attr, NULL);
4111}
4112
4113static inline void netdev_class_remove_file(const struct class_attribute *class_attr)
4114{
4115 netdev_class_remove_file_ns(class_attr, NULL);
4116}
4117
4118extern const struct kobj_ns_type_operations net_ns_type_operations;
4119
4120const char *netdev_drivername(const struct net_device *dev);
4121
4122void linkwatch_run_queue(void);
4123
4124static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4125 netdev_features_t f2)
4126{
4127 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4128 if (f1 & NETIF_F_HW_CSUM)
4129 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4130 else
4131 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4132 }
4133
4134 return f1 & f2;
4135}
4136
4137static inline netdev_features_t netdev_get_wanted_features(
4138 struct net_device *dev)
4139{
4140 return (dev->features & ~dev->hw_features) | dev->wanted_features;
4141}
4142netdev_features_t netdev_increment_features(netdev_features_t all,
4143 netdev_features_t one, netdev_features_t mask);
4144
4145
4146
4147
4148
4149static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4150 netdev_features_t mask)
4151{
4152 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4153}
4154
4155int __netdev_update_features(struct net_device *dev);
4156void netdev_update_features(struct net_device *dev);
4157void netdev_change_features(struct net_device *dev);
4158
4159void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4160 struct net_device *dev);
4161
4162netdev_features_t passthru_features_check(struct sk_buff *skb,
4163 struct net_device *dev,
4164 netdev_features_t features);
4165netdev_features_t netif_skb_features(struct sk_buff *skb);
4166
4167static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4168{
4169 netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
4170
4171
4172 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
4173 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
4174 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
4175 BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
4176 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
4177 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
4178 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
4179 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
4180 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
4181 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
4182 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
4183 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
4184 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
4185 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
4186 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
4187 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
4188 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
4189
4190 return (features & feature) == feature;
4191}
4192
4193static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
4194{
4195 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
4196 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
4197}
4198
4199static inline bool netif_needs_gso(struct sk_buff *skb,
4200 netdev_features_t features)
4201{
4202 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
4203 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
4204 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
4205}
4206
4207static inline void netif_set_gso_max_size(struct net_device *dev,
4208 unsigned int size)
4209{
4210 dev->gso_max_size = size;
4211}
4212
4213static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
4214 int pulled_hlen, u16 mac_offset,
4215 int mac_len)
4216{
4217 skb->protocol = protocol;
4218 skb->encapsulation = 1;
4219 skb_push(skb, pulled_hlen);
4220 skb_reset_transport_header(skb);
4221 skb->mac_header = mac_offset;
4222 skb->network_header = skb->mac_header + mac_len;
4223 skb->mac_len = mac_len;
4224}
4225
4226static inline bool netif_is_macsec(const struct net_device *dev)
4227{
4228 return dev->priv_flags & IFF_MACSEC;
4229}
4230
4231static inline bool netif_is_macvlan(const struct net_device *dev)
4232{
4233 return dev->priv_flags & IFF_MACVLAN;
4234}
4235
4236static inline bool netif_is_macvlan_port(const struct net_device *dev)
4237{
4238 return dev->priv_flags & IFF_MACVLAN_PORT;
4239}
4240
4241static inline bool netif_is_bond_master(const struct net_device *dev)
4242{
4243 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
4244}
4245
4246static inline bool netif_is_bond_slave(const struct net_device *dev)
4247{
4248 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
4249}
4250
4251static inline bool netif_supports_nofcs(struct net_device *dev)
4252{
4253 return dev->priv_flags & IFF_SUPP_NOFCS;
4254}
4255
4256static inline bool netif_is_l3_master(const struct net_device *dev)
4257{
4258 return dev->priv_flags & IFF_L3MDEV_MASTER;
4259}
4260
4261static inline bool netif_is_l3_slave(const struct net_device *dev)
4262{
4263 return dev->priv_flags & IFF_L3MDEV_SLAVE;
4264}
4265
4266static inline bool netif_is_bridge_master(const struct net_device *dev)
4267{
4268 return dev->priv_flags & IFF_EBRIDGE;
4269}
4270
4271static inline bool netif_is_bridge_port(const struct net_device *dev)
4272{
4273 return dev->priv_flags & IFF_BRIDGE_PORT;
4274}
4275
4276static inline bool netif_is_ovs_master(const struct net_device *dev)
4277{
4278 return dev->priv_flags & IFF_OPENVSWITCH;
4279}
4280
4281static inline bool netif_is_ovs_port(const struct net_device *dev)
4282{
4283 return dev->priv_flags & IFF_OVS_DATAPATH;
4284}
4285
4286static inline bool netif_is_team_master(const struct net_device *dev)
4287{
4288 return dev->priv_flags & IFF_TEAM;
4289}
4290
4291static inline bool netif_is_team_port(const struct net_device *dev)
4292{
4293 return dev->priv_flags & IFF_TEAM_PORT;
4294}
4295
4296static inline bool netif_is_lag_master(const struct net_device *dev)
4297{
4298 return netif_is_bond_master(dev) || netif_is_team_master(dev);
4299}
4300
4301static inline bool netif_is_lag_port(const struct net_device *dev)
4302{
4303 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
4304}
4305
4306static inline bool netif_is_rxfh_configured(const struct net_device *dev)
4307{
4308 return dev->priv_flags & IFF_RXFH_CONFIGURED;
4309}
4310
4311
4312static inline void netif_keep_dst(struct net_device *dev)
4313{
4314 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
4315}
4316
4317
4318static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
4319{
4320
4321 return dev->priv_flags & IFF_MACSEC;
4322}
4323
4324extern struct pernet_operations __net_initdata loopback_net_ops;
4325
4326
4327
4328
4329
4330static inline const char *netdev_name(const struct net_device *dev)
4331{
4332 if (!dev->name[0] || strchr(dev->name, '%'))
4333 return "(unnamed net_device)";
4334 return dev->name;
4335}
4336
4337static inline bool netdev_unregistering(const struct net_device *dev)
4338{
4339 return dev->reg_state == NETREG_UNREGISTERING;
4340}
4341
4342static inline const char *netdev_reg_state(const struct net_device *dev)
4343{
4344 switch (dev->reg_state) {
4345 case NETREG_UNINITIALIZED: return " (uninitialized)";
4346 case NETREG_REGISTERED: return "";
4347 case NETREG_UNREGISTERING: return " (unregistering)";
4348 case NETREG_UNREGISTERED: return " (unregistered)";
4349 case NETREG_RELEASED: return " (released)";
4350 case NETREG_DUMMY: return " (dummy)";
4351 }
4352
4353 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
4354 return " (unknown)";
4355}
4356
4357__printf(3, 4)
4358void netdev_printk(const char *level, const struct net_device *dev,
4359 const char *format, ...);
4360__printf(2, 3)
4361void netdev_emerg(const struct net_device *dev, const char *format, ...);
4362__printf(2, 3)
4363void netdev_alert(const struct net_device *dev, const char *format, ...);
4364__printf(2, 3)
4365void netdev_crit(const struct net_device *dev, const char *format, ...);
4366__printf(2, 3)
4367void netdev_err(const struct net_device *dev, const char *format, ...);
4368__printf(2, 3)
4369void netdev_warn(const struct net_device *dev, const char *format, ...);
4370__printf(2, 3)
4371void netdev_notice(const struct net_device *dev, const char *format, ...);
4372__printf(2, 3)
4373void netdev_info(const struct net_device *dev, const char *format, ...);
4374
4375#define netdev_level_once(level, dev, fmt, ...) \
4376do { \
4377 static bool __print_once __read_mostly; \
4378 \
4379 if (!__print_once) { \
4380 __print_once = true; \
4381 netdev_printk(level, dev, fmt, ##__VA_ARGS__); \
4382 } \
4383} while (0)
4384
4385#define netdev_emerg_once(dev, fmt, ...) \
4386 netdev_level_once(KERN_EMERG, dev, fmt, ##__VA_ARGS__)
4387#define netdev_alert_once(dev, fmt, ...) \
4388 netdev_level_once(KERN_ALERT, dev, fmt, ##__VA_ARGS__)
4389#define netdev_crit_once(dev, fmt, ...) \
4390 netdev_level_once(KERN_CRIT, dev, fmt, ##__VA_ARGS__)
4391#define netdev_err_once(dev, fmt, ...) \
4392 netdev_level_once(KERN_ERR, dev, fmt, ##__VA_ARGS__)
4393#define netdev_warn_once(dev, fmt, ...) \
4394 netdev_level_once(KERN_WARNING, dev, fmt, ##__VA_ARGS__)
4395#define netdev_notice_once(dev, fmt, ...) \
4396 netdev_level_once(KERN_NOTICE, dev, fmt, ##__VA_ARGS__)
4397#define netdev_info_once(dev, fmt, ...) \
4398 netdev_level_once(KERN_INFO, dev, fmt, ##__VA_ARGS__)
4399
4400#define MODULE_ALIAS_NETDEV(device) \
4401 MODULE_ALIAS("netdev-" device)
4402
4403#if defined(CONFIG_DYNAMIC_DEBUG)
4404#define netdev_dbg(__dev, format, args...) \
4405do { \
4406 dynamic_netdev_dbg(__dev, format, ##args); \
4407} while (0)
4408#elif defined(DEBUG)
4409#define netdev_dbg(__dev, format, args...) \
4410 netdev_printk(KERN_DEBUG, __dev, format, ##args)
4411#else
4412#define netdev_dbg(__dev, format, args...) \
4413({ \
4414 if (0) \
4415 netdev_printk(KERN_DEBUG, __dev, format, ##args); \
4416})
4417#endif
4418
4419#if defined(VERBOSE_DEBUG)
4420#define netdev_vdbg netdev_dbg
4421#else
4422
4423#define netdev_vdbg(dev, format, args...) \
4424({ \
4425 if (0) \
4426 netdev_printk(KERN_DEBUG, dev, format, ##args); \
4427 0; \
4428})
4429#endif
4430
4431
4432
4433
4434
4435
4436#define netdev_WARN(dev, format, args...) \
4437 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
4438 netdev_reg_state(dev), ##args)
4439
4440#define netdev_WARN_ONCE(dev, format, args...) \
4441 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
4442 netdev_reg_state(dev), ##args)
4443
4444
4445
4446#define netif_printk(priv, type, level, dev, fmt, args...) \
4447do { \
4448 if (netif_msg_##type(priv)) \
4449 netdev_printk(level, (dev), fmt, ##args); \
4450} while (0)
4451
4452#define netif_level(level, priv, type, dev, fmt, args...) \
4453do { \
4454 if (netif_msg_##type(priv)) \
4455 netdev_##level(dev, fmt, ##args); \
4456} while (0)
4457
4458#define netif_emerg(priv, type, dev, fmt, args...) \
4459 netif_level(emerg, priv, type, dev, fmt, ##args)
4460#define netif_alert(priv, type, dev, fmt, args...) \
4461 netif_level(alert, priv, type, dev, fmt, ##args)
4462#define netif_crit(priv, type, dev, fmt, args...) \
4463 netif_level(crit, priv, type, dev, fmt, ##args)
4464#define netif_err(priv, type, dev, fmt, args...) \
4465 netif_level(err, priv, type, dev, fmt, ##args)
4466#define netif_warn(priv, type, dev, fmt, args...) \
4467 netif_level(warn, priv, type, dev, fmt, ##args)
4468#define netif_notice(priv, type, dev, fmt, args...) \
4469 netif_level(notice, priv, type, dev, fmt, ##args)
4470#define netif_info(priv, type, dev, fmt, args...) \
4471 netif_level(info, priv, type, dev, fmt, ##args)
4472
4473#if defined(CONFIG_DYNAMIC_DEBUG)
4474#define netif_dbg(priv, type, netdev, format, args...) \
4475do { \
4476 if (netif_msg_##type(priv)) \
4477 dynamic_netdev_dbg(netdev, format, ##args); \
4478} while (0)
4479#elif defined(DEBUG)
4480#define netif_dbg(priv, type, dev, format, args...) \
4481 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
4482#else
4483#define netif_dbg(priv, type, dev, format, args...) \
4484({ \
4485 if (0) \
4486 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4487 0; \
4488})
4489#endif
4490
4491
4492#define netif_cond_dbg(priv, type, netdev, cond, level, fmt, args...) \
4493 do { \
4494 if (cond) \
4495 netif_dbg(priv, type, netdev, fmt, ##args); \
4496 else \
4497 netif_ ## level(priv, type, netdev, fmt, ##args); \
4498 } while (0)
4499
4500#if defined(VERBOSE_DEBUG)
4501#define netif_vdbg netif_dbg
4502#else
4503#define netif_vdbg(priv, type, dev, format, args...) \
4504({ \
4505 if (0) \
4506 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4507 0; \
4508})
4509#endif
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530#define PTYPE_HASH_SIZE (16)
4531#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
4532
4533#endif
4534