1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef _LINUX_NETDEVICE_H
26#define _LINUX_NETDEVICE_H
27
28#include <linux/timer.h>
29#include <linux/bug.h>
30#include <linux/delay.h>
31#include <linux/atomic.h>
32#include <linux/prefetch.h>
33#include <asm/cache.h>
34#include <asm/byteorder.h>
35
36#include <linux/percpu.h>
37#include <linux/rculist.h>
38#include <linux/workqueue.h>
39#include <linux/dynamic_queue_limits.h>
40
41#include <linux/ethtool.h>
42#include <net/net_namespace.h>
43#ifdef CONFIG_DCB
44#include <net/dcbnl.h>
45#endif
46#include <net/netprio_cgroup.h>
47#include <net/xdp.h>
48
49#include <linux/netdev_features.h>
50#include <linux/neighbour.h>
51#include <uapi/linux/netdevice.h>
52#include <uapi/linux/if_bonding.h>
53#include <uapi/linux/pkt_cls.h>
54#include <linux/hashtable.h>
55
56struct netpoll_info;
57struct device;
58struct phy_device;
59struct dsa_port;
60
61
62struct wireless_dev;
63
64struct wpan_dev;
65struct mpls_dev;
66
67struct udp_tunnel_info;
68struct bpf_prog;
69struct xdp_buff;
70
71void netdev_set_default_ethtool_ops(struct net_device *dev,
72 const struct ethtool_ops *ops);
73
74
75#define NET_RX_SUCCESS 0
76#define NET_RX_DROP 1
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96#define NET_XMIT_SUCCESS 0x00
97#define NET_XMIT_DROP 0x01
98#define NET_XMIT_CN 0x02
99#define NET_XMIT_MASK 0x0f
100
101
102
103
104#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
105#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
106
107
108#define NETDEV_TX_MASK 0xf0
109
110enum netdev_tx {
111 __NETDEV_TX_MIN = INT_MIN,
112 NETDEV_TX_OK = 0x00,
113 NETDEV_TX_BUSY = 0x10,
114};
115typedef enum netdev_tx netdev_tx_t;
116
117
118
119
120
121static inline bool dev_xmit_complete(int rc)
122{
123
124
125
126
127
128
129 if (likely(rc < NET_XMIT_MASK))
130 return true;
131
132 return false;
133}
134
135
136
137
138
139
140#if defined(CONFIG_HYPERV_NET)
141# define LL_MAX_HEADER 128
142#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
143# if defined(CONFIG_MAC80211_MESH)
144# define LL_MAX_HEADER 128
145# else
146# define LL_MAX_HEADER 96
147# endif
148#else
149# define LL_MAX_HEADER 32
150#endif
151
152#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
153 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
154#define MAX_HEADER LL_MAX_HEADER
155#else
156#define MAX_HEADER (LL_MAX_HEADER + 48)
157#endif
158
159
160
161
162
163
164struct net_device_stats {
165 unsigned long rx_packets;
166 unsigned long tx_packets;
167 unsigned long rx_bytes;
168 unsigned long tx_bytes;
169 unsigned long rx_errors;
170 unsigned long tx_errors;
171 unsigned long rx_dropped;
172 unsigned long tx_dropped;
173 unsigned long multicast;
174 unsigned long collisions;
175 unsigned long rx_length_errors;
176 unsigned long rx_over_errors;
177 unsigned long rx_crc_errors;
178 unsigned long rx_frame_errors;
179 unsigned long rx_fifo_errors;
180 unsigned long rx_missed_errors;
181 unsigned long tx_aborted_errors;
182 unsigned long tx_carrier_errors;
183 unsigned long tx_fifo_errors;
184 unsigned long tx_heartbeat_errors;
185 unsigned long tx_window_errors;
186 unsigned long rx_compressed;
187 unsigned long tx_compressed;
188};
189
190
191#include <linux/cache.h>
192#include <linux/skbuff.h>
193
194#ifdef CONFIG_RPS
195#include <linux/static_key.h>
196extern struct static_key rps_needed;
197extern struct static_key rfs_needed;
198#endif
199
200struct neighbour;
201struct neigh_parms;
202struct sk_buff;
203
204struct netdev_hw_addr {
205 struct list_head list;
206 unsigned char addr[MAX_ADDR_LEN];
207 unsigned char type;
208#define NETDEV_HW_ADDR_T_LAN 1
209#define NETDEV_HW_ADDR_T_SAN 2
210#define NETDEV_HW_ADDR_T_SLAVE 3
211#define NETDEV_HW_ADDR_T_UNICAST 4
212#define NETDEV_HW_ADDR_T_MULTICAST 5
213 bool global_use;
214 int sync_cnt;
215 int refcount;
216 int synced;
217 struct rcu_head rcu_head;
218};
219
220struct netdev_hw_addr_list {
221 struct list_head list;
222 int count;
223};
224
225#define netdev_hw_addr_list_count(l) ((l)->count)
226#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
227#define netdev_hw_addr_list_for_each(ha, l) \
228 list_for_each_entry(ha, &(l)->list, list)
229
230#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
231#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
232#define netdev_for_each_uc_addr(ha, dev) \
233 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
234
235#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
236#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
237#define netdev_for_each_mc_addr(ha, dev) \
238 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
239
240struct hh_cache {
241 unsigned int hh_len;
242 seqlock_t hh_lock;
243
244
245#define HH_DATA_MOD 16
246#define HH_DATA_OFF(__len) \
247 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
248#define HH_DATA_ALIGN(__len) \
249 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
250 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
251};
252
253
254
255
256
257
258
259
260
261#define LL_RESERVED_SPACE(dev) \
262 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
263#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
264 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
265
266struct header_ops {
267 int (*create) (struct sk_buff *skb, struct net_device *dev,
268 unsigned short type, const void *daddr,
269 const void *saddr, unsigned int len);
270 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
271 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
272 void (*cache_update)(struct hh_cache *hh,
273 const struct net_device *dev,
274 const unsigned char *haddr);
275 bool (*validate)(const char *ll_header, unsigned int len);
276};
277
278
279
280
281
282
283enum netdev_state_t {
284 __LINK_STATE_START,
285 __LINK_STATE_PRESENT,
286 __LINK_STATE_NOCARRIER,
287 __LINK_STATE_LINKWATCH_PENDING,
288 __LINK_STATE_DORMANT,
289};
290
291
292
293
294
295
296struct netdev_boot_setup {
297 char name[IFNAMSIZ];
298 struct ifmap map;
299};
300#define NETDEV_BOOT_SETUP_MAX 8
301
302int __init netdev_boot_setup(char *str);
303
304
305
306
307struct napi_struct {
308
309
310
311
312
313
314 struct list_head poll_list;
315
316 unsigned long state;
317 int weight;
318 unsigned int gro_count;
319 int (*poll)(struct napi_struct *, int);
320#ifdef CONFIG_NETPOLL
321 int poll_owner;
322#endif
323 struct net_device *dev;
324 struct sk_buff *gro_list;
325 struct sk_buff *skb;
326 struct hrtimer timer;
327 struct list_head dev_list;
328 struct hlist_node napi_hash_node;
329 unsigned int napi_id;
330};
331
332enum {
333 NAPI_STATE_SCHED,
334 NAPI_STATE_MISSED,
335 NAPI_STATE_DISABLE,
336 NAPI_STATE_NPSVC,
337 NAPI_STATE_HASHED,
338 NAPI_STATE_NO_BUSY_POLL,
339 NAPI_STATE_IN_BUSY_POLL,
340};
341
342enum {
343 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
344 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
345 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
346 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
347 NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED),
348 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
349 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
350};
351
352enum gro_result {
353 GRO_MERGED,
354 GRO_MERGED_FREE,
355 GRO_HELD,
356 GRO_NORMAL,
357 GRO_DROP,
358 GRO_CONSUMED,
359};
360typedef enum gro_result gro_result_t;
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403enum rx_handler_result {
404 RX_HANDLER_CONSUMED,
405 RX_HANDLER_ANOTHER,
406 RX_HANDLER_EXACT,
407 RX_HANDLER_PASS,
408};
409typedef enum rx_handler_result rx_handler_result_t;
410typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
411
412void __napi_schedule(struct napi_struct *n);
413void __napi_schedule_irqoff(struct napi_struct *n);
414
415static inline bool napi_disable_pending(struct napi_struct *n)
416{
417 return test_bit(NAPI_STATE_DISABLE, &n->state);
418}
419
420bool napi_schedule_prep(struct napi_struct *n);
421
422
423
424
425
426
427
428
429static inline void napi_schedule(struct napi_struct *n)
430{
431 if (napi_schedule_prep(n))
432 __napi_schedule(n);
433}
434
435
436
437
438
439
440
441static inline void napi_schedule_irqoff(struct napi_struct *n)
442{
443 if (napi_schedule_prep(n))
444 __napi_schedule_irqoff(n);
445}
446
447
448static inline bool napi_reschedule(struct napi_struct *napi)
449{
450 if (napi_schedule_prep(napi)) {
451 __napi_schedule(napi);
452 return true;
453 }
454 return false;
455}
456
457bool napi_complete_done(struct napi_struct *n, int work_done);
458
459
460
461
462
463
464
465
466static inline bool napi_complete(struct napi_struct *n)
467{
468 return napi_complete_done(n, 0);
469}
470
471
472
473
474
475
476
477
478
479
480
481
482
483bool napi_hash_del(struct napi_struct *napi);
484
485
486
487
488
489
490
491
492void napi_disable(struct napi_struct *n);
493
494
495
496
497
498
499
500
501static inline void napi_enable(struct napi_struct *n)
502{
503 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
504 smp_mb__before_atomic();
505 clear_bit(NAPI_STATE_SCHED, &n->state);
506 clear_bit(NAPI_STATE_NPSVC, &n->state);
507}
508
509
510
511
512
513
514
515
516
517static inline void napi_synchronize(const struct napi_struct *n)
518{
519 if (IS_ENABLED(CONFIG_SMP))
520 while (test_bit(NAPI_STATE_SCHED, &n->state))
521 msleep(1);
522 else
523 barrier();
524}
525
526enum netdev_queue_state_t {
527 __QUEUE_STATE_DRV_XOFF,
528 __QUEUE_STATE_STACK_XOFF,
529 __QUEUE_STATE_FROZEN,
530};
531
532#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
533#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
534#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
535
536#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
537#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
538 QUEUE_STATE_FROZEN)
539#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
540 QUEUE_STATE_FROZEN)
541
542
543
544
545
546
547
548
549
550
551
552struct netdev_queue {
553
554
555
556 struct net_device *dev;
557 struct Qdisc __rcu *qdisc;
558 struct Qdisc *qdisc_sleeping;
559#ifdef CONFIG_SYSFS
560 struct kobject kobj;
561#endif
562#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
563 int numa_node;
564#endif
565 unsigned long tx_maxrate;
566
567
568
569
570 unsigned long trans_timeout;
571
572
573
574 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
575 int xmit_lock_owner;
576
577
578
579 unsigned long trans_start;
580
581 unsigned long state;
582
583#ifdef CONFIG_BQL
584 struct dql dql;
585#endif
586} ____cacheline_aligned_in_smp;
587
588static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
589{
590#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
591 return q->numa_node;
592#else
593 return NUMA_NO_NODE;
594#endif
595}
596
597static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
598{
599#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
600 q->numa_node = node;
601#endif
602}
603
604#ifdef CONFIG_RPS
605
606
607
608
609struct rps_map {
610 unsigned int len;
611 struct rcu_head rcu;
612 u16 cpus[0];
613};
614#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
615
616
617
618
619
620
621struct rps_dev_flow {
622 u16 cpu;
623 u16 filter;
624 unsigned int last_qtail;
625};
626#define RPS_NO_FILTER 0xffff
627
628
629
630
631struct rps_dev_flow_table {
632 unsigned int mask;
633 struct rcu_head rcu;
634 struct rps_dev_flow flows[0];
635};
636#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
637 ((_num) * sizeof(struct rps_dev_flow)))
638
639
640
641
642
643
644
645
646
647
648
649struct rps_sock_flow_table {
650 u32 mask;
651
652 u32 ents[0] ____cacheline_aligned_in_smp;
653};
654#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
655
656#define RPS_NO_CPU 0xffff
657
658extern u32 rps_cpu_mask;
659extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
660
661static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
662 u32 hash)
663{
664 if (table && hash) {
665 unsigned int index = hash & table->mask;
666 u32 val = hash & ~rps_cpu_mask;
667
668
669 val |= raw_smp_processor_id();
670
671 if (table->ents[index] != val)
672 table->ents[index] = val;
673 }
674}
675
676#ifdef CONFIG_RFS_ACCEL
677bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
678 u16 filter_id);
679#endif
680#endif
681
682
683struct netdev_rx_queue {
684#ifdef CONFIG_RPS
685 struct rps_map __rcu *rps_map;
686 struct rps_dev_flow_table __rcu *rps_flow_table;
687#endif
688 struct kobject kobj;
689 struct net_device *dev;
690 struct xdp_rxq_info xdp_rxq;
691} ____cacheline_aligned_in_smp;
692
693
694
695
696struct rx_queue_attribute {
697 struct attribute attr;
698 ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
699 ssize_t (*store)(struct netdev_rx_queue *queue,
700 const char *buf, size_t len);
701};
702
703#ifdef CONFIG_XPS
704
705
706
707
708struct xps_map {
709 unsigned int len;
710 unsigned int alloc_len;
711 struct rcu_head rcu;
712 u16 queues[0];
713};
714#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
715#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
716 - sizeof(struct xps_map)) / sizeof(u16))
717
718
719
720
721struct xps_dev_maps {
722 struct rcu_head rcu;
723 struct xps_map __rcu *cpu_map[0];
724};
725#define XPS_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
726 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
727#endif
728
729#define TC_MAX_QUEUE 16
730#define TC_BITMASK 15
731
732struct netdev_tc_txq {
733 u16 count;
734 u16 offset;
735};
736
737#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
738
739
740
741
742struct netdev_fcoe_hbainfo {
743 char manufacturer[64];
744 char serial_number[64];
745 char hardware_version[64];
746 char driver_version[64];
747 char optionrom_version[64];
748 char firmware_version[64];
749 char model[256];
750 char model_description[256];
751};
752#endif
753
754#define MAX_PHYS_ITEM_ID_LEN 32
755
756
757
758
759struct netdev_phys_item_id {
760 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
761 unsigned char id_len;
762};
763
764static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
765 struct netdev_phys_item_id *b)
766{
767 return a->id_len == b->id_len &&
768 memcmp(a->id, b->id, a->id_len) == 0;
769}
770
771typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
772 struct sk_buff *skb);
773
774enum tc_setup_type {
775 TC_SETUP_QDISC_MQPRIO,
776 TC_SETUP_CLSU32,
777 TC_SETUP_CLSFLOWER,
778 TC_SETUP_CLSMATCHALL,
779 TC_SETUP_CLSBPF,
780 TC_SETUP_BLOCK,
781 TC_SETUP_QDISC_CBS,
782 TC_SETUP_QDISC_RED,
783 TC_SETUP_QDISC_PRIO,
784};
785
786
787
788
789enum bpf_netdev_command {
790
791
792
793
794
795
796
797 XDP_SETUP_PROG,
798 XDP_SETUP_PROG_HW,
799
800
801
802
803 XDP_QUERY_PROG,
804
805 BPF_OFFLOAD_VERIFIER_PREP,
806 BPF_OFFLOAD_TRANSLATE,
807 BPF_OFFLOAD_DESTROY,
808 BPF_OFFLOAD_MAP_ALLOC,
809 BPF_OFFLOAD_MAP_FREE,
810};
811
812struct bpf_prog_offload_ops;
813struct netlink_ext_ack;
814
815struct netdev_bpf {
816 enum bpf_netdev_command command;
817 union {
818
819 struct {
820 u32 flags;
821 struct bpf_prog *prog;
822 struct netlink_ext_ack *extack;
823 };
824
825 struct {
826 u8 prog_attached;
827 u32 prog_id;
828
829 u32 prog_flags;
830 };
831
832 struct {
833 struct bpf_prog *prog;
834 const struct bpf_prog_offload_ops *ops;
835 } verifier;
836
837 struct {
838 struct bpf_prog *prog;
839 } offload;
840
841 struct {
842 struct bpf_offloaded_map *offmap;
843 };
844 };
845};
846
847#ifdef CONFIG_XFRM_OFFLOAD
848struct xfrmdev_ops {
849 int (*xdo_dev_state_add) (struct xfrm_state *x);
850 void (*xdo_dev_state_delete) (struct xfrm_state *x);
851 void (*xdo_dev_state_free) (struct xfrm_state *x);
852 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
853 struct xfrm_state *x);
854 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
855};
856#endif
857
858struct dev_ifalias {
859 struct rcu_head rcuhead;
860 char ifalias[];
861};
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165struct net_device_ops {
1166 int (*ndo_init)(struct net_device *dev);
1167 void (*ndo_uninit)(struct net_device *dev);
1168 int (*ndo_open)(struct net_device *dev);
1169 int (*ndo_stop)(struct net_device *dev);
1170 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1171 struct net_device *dev);
1172 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1173 struct net_device *dev,
1174 netdev_features_t features);
1175 u16 (*ndo_select_queue)(struct net_device *dev,
1176 struct sk_buff *skb,
1177 void *accel_priv,
1178 select_queue_fallback_t fallback);
1179 void (*ndo_change_rx_flags)(struct net_device *dev,
1180 int flags);
1181 void (*ndo_set_rx_mode)(struct net_device *dev);
1182 int (*ndo_set_mac_address)(struct net_device *dev,
1183 void *addr);
1184 int (*ndo_validate_addr)(struct net_device *dev);
1185 int (*ndo_do_ioctl)(struct net_device *dev,
1186 struct ifreq *ifr, int cmd);
1187 int (*ndo_set_config)(struct net_device *dev,
1188 struct ifmap *map);
1189 int (*ndo_change_mtu)(struct net_device *dev,
1190 int new_mtu);
1191 int (*ndo_neigh_setup)(struct net_device *dev,
1192 struct neigh_parms *);
1193 void (*ndo_tx_timeout) (struct net_device *dev);
1194
1195 void (*ndo_get_stats64)(struct net_device *dev,
1196 struct rtnl_link_stats64 *storage);
1197 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1198 int (*ndo_get_offload_stats)(int attr_id,
1199 const struct net_device *dev,
1200 void *attr_data);
1201 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1202
1203 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1204 __be16 proto, u16 vid);
1205 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1206 __be16 proto, u16 vid);
1207#ifdef CONFIG_NET_POLL_CONTROLLER
1208 void (*ndo_poll_controller)(struct net_device *dev);
1209 int (*ndo_netpoll_setup)(struct net_device *dev,
1210 struct netpoll_info *info);
1211 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1212#endif
1213 int (*ndo_set_vf_mac)(struct net_device *dev,
1214 int queue, u8 *mac);
1215 int (*ndo_set_vf_vlan)(struct net_device *dev,
1216 int queue, u16 vlan,
1217 u8 qos, __be16 proto);
1218 int (*ndo_set_vf_rate)(struct net_device *dev,
1219 int vf, int min_tx_rate,
1220 int max_tx_rate);
1221 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1222 int vf, bool setting);
1223 int (*ndo_set_vf_trust)(struct net_device *dev,
1224 int vf, bool setting);
1225 int (*ndo_get_vf_config)(struct net_device *dev,
1226 int vf,
1227 struct ifla_vf_info *ivf);
1228 int (*ndo_set_vf_link_state)(struct net_device *dev,
1229 int vf, int link_state);
1230 int (*ndo_get_vf_stats)(struct net_device *dev,
1231 int vf,
1232 struct ifla_vf_stats
1233 *vf_stats);
1234 int (*ndo_set_vf_port)(struct net_device *dev,
1235 int vf,
1236 struct nlattr *port[]);
1237 int (*ndo_get_vf_port)(struct net_device *dev,
1238 int vf, struct sk_buff *skb);
1239 int (*ndo_set_vf_guid)(struct net_device *dev,
1240 int vf, u64 guid,
1241 int guid_type);
1242 int (*ndo_set_vf_rss_query_en)(
1243 struct net_device *dev,
1244 int vf, bool setting);
1245 int (*ndo_setup_tc)(struct net_device *dev,
1246 enum tc_setup_type type,
1247 void *type_data);
1248#if IS_ENABLED(CONFIG_FCOE)
1249 int (*ndo_fcoe_enable)(struct net_device *dev);
1250 int (*ndo_fcoe_disable)(struct net_device *dev);
1251 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1252 u16 xid,
1253 struct scatterlist *sgl,
1254 unsigned int sgc);
1255 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1256 u16 xid);
1257 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1258 u16 xid,
1259 struct scatterlist *sgl,
1260 unsigned int sgc);
1261 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1262 struct netdev_fcoe_hbainfo *hbainfo);
1263#endif
1264
1265#if IS_ENABLED(CONFIG_LIBFCOE)
1266#define NETDEV_FCOE_WWNN 0
1267#define NETDEV_FCOE_WWPN 1
1268 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1269 u64 *wwn, int type);
1270#endif
1271
1272#ifdef CONFIG_RFS_ACCEL
1273 int (*ndo_rx_flow_steer)(struct net_device *dev,
1274 const struct sk_buff *skb,
1275 u16 rxq_index,
1276 u32 flow_id);
1277#endif
1278 int (*ndo_add_slave)(struct net_device *dev,
1279 struct net_device *slave_dev,
1280 struct netlink_ext_ack *extack);
1281 int (*ndo_del_slave)(struct net_device *dev,
1282 struct net_device *slave_dev);
1283 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1284 netdev_features_t features);
1285 int (*ndo_set_features)(struct net_device *dev,
1286 netdev_features_t features);
1287 int (*ndo_neigh_construct)(struct net_device *dev,
1288 struct neighbour *n);
1289 void (*ndo_neigh_destroy)(struct net_device *dev,
1290 struct neighbour *n);
1291
1292 int (*ndo_fdb_add)(struct ndmsg *ndm,
1293 struct nlattr *tb[],
1294 struct net_device *dev,
1295 const unsigned char *addr,
1296 u16 vid,
1297 u16 flags);
1298 int (*ndo_fdb_del)(struct ndmsg *ndm,
1299 struct nlattr *tb[],
1300 struct net_device *dev,
1301 const unsigned char *addr,
1302 u16 vid);
1303 int (*ndo_fdb_dump)(struct sk_buff *skb,
1304 struct netlink_callback *cb,
1305 struct net_device *dev,
1306 struct net_device *filter_dev,
1307 int *idx);
1308
1309 int (*ndo_bridge_setlink)(struct net_device *dev,
1310 struct nlmsghdr *nlh,
1311 u16 flags);
1312 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1313 u32 pid, u32 seq,
1314 struct net_device *dev,
1315 u32 filter_mask,
1316 int nlflags);
1317 int (*ndo_bridge_dellink)(struct net_device *dev,
1318 struct nlmsghdr *nlh,
1319 u16 flags);
1320 int (*ndo_change_carrier)(struct net_device *dev,
1321 bool new_carrier);
1322 int (*ndo_get_phys_port_id)(struct net_device *dev,
1323 struct netdev_phys_item_id *ppid);
1324 int (*ndo_get_phys_port_name)(struct net_device *dev,
1325 char *name, size_t len);
1326 void (*ndo_udp_tunnel_add)(struct net_device *dev,
1327 struct udp_tunnel_info *ti);
1328 void (*ndo_udp_tunnel_del)(struct net_device *dev,
1329 struct udp_tunnel_info *ti);
1330 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1331 struct net_device *dev);
1332 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1333 void *priv);
1334
1335 int (*ndo_get_lock_subclass)(struct net_device *dev);
1336 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1337 int queue_index,
1338 u32 maxrate);
1339 int (*ndo_get_iflink)(const struct net_device *dev);
1340 int (*ndo_change_proto_down)(struct net_device *dev,
1341 bool proto_down);
1342 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1343 struct sk_buff *skb);
1344 void (*ndo_set_rx_headroom)(struct net_device *dev,
1345 int needed_headroom);
1346 int (*ndo_bpf)(struct net_device *dev,
1347 struct netdev_bpf *bpf);
1348 int (*ndo_xdp_xmit)(struct net_device *dev,
1349 struct xdp_buff *xdp);
1350 void (*ndo_xdp_flush)(struct net_device *dev);
1351};
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396enum netdev_priv_flags {
1397 IFF_802_1Q_VLAN = 1<<0,
1398 IFF_EBRIDGE = 1<<1,
1399 IFF_BONDING = 1<<2,
1400 IFF_ISATAP = 1<<3,
1401 IFF_WAN_HDLC = 1<<4,
1402 IFF_XMIT_DST_RELEASE = 1<<5,
1403 IFF_DONT_BRIDGE = 1<<6,
1404 IFF_DISABLE_NETPOLL = 1<<7,
1405 IFF_MACVLAN_PORT = 1<<8,
1406 IFF_BRIDGE_PORT = 1<<9,
1407 IFF_OVS_DATAPATH = 1<<10,
1408 IFF_TX_SKB_SHARING = 1<<11,
1409 IFF_UNICAST_FLT = 1<<12,
1410 IFF_TEAM_PORT = 1<<13,
1411 IFF_SUPP_NOFCS = 1<<14,
1412 IFF_LIVE_ADDR_CHANGE = 1<<15,
1413 IFF_MACVLAN = 1<<16,
1414 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1415 IFF_IPVLAN_MASTER = 1<<18,
1416 IFF_IPVLAN_SLAVE = 1<<19,
1417 IFF_L3MDEV_MASTER = 1<<20,
1418 IFF_NO_QUEUE = 1<<21,
1419 IFF_OPENVSWITCH = 1<<22,
1420 IFF_L3MDEV_SLAVE = 1<<23,
1421 IFF_TEAM = 1<<24,
1422 IFF_RXFH_CONFIGURED = 1<<25,
1423 IFF_PHONY_HEADROOM = 1<<26,
1424 IFF_MACSEC = 1<<27,
1425};
1426
1427#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
1428#define IFF_EBRIDGE IFF_EBRIDGE
1429#define IFF_BONDING IFF_BONDING
1430#define IFF_ISATAP IFF_ISATAP
1431#define IFF_WAN_HDLC IFF_WAN_HDLC
1432#define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
1433#define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
1434#define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
1435#define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
1436#define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
1437#define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
1438#define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
1439#define IFF_UNICAST_FLT IFF_UNICAST_FLT
1440#define IFF_TEAM_PORT IFF_TEAM_PORT
1441#define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
1442#define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
1443#define IFF_MACVLAN IFF_MACVLAN
1444#define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
1445#define IFF_IPVLAN_MASTER IFF_IPVLAN_MASTER
1446#define IFF_IPVLAN_SLAVE IFF_IPVLAN_SLAVE
1447#define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
1448#define IFF_NO_QUEUE IFF_NO_QUEUE
1449#define IFF_OPENVSWITCH IFF_OPENVSWITCH
1450#define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
1451#define IFF_TEAM IFF_TEAM
1452#define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED
1453#define IFF_MACSEC IFF_MACSEC
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671struct net_device {
1672 char name[IFNAMSIZ];
1673 struct hlist_node name_hlist;
1674 struct dev_ifalias __rcu *ifalias;
1675
1676
1677
1678
1679 unsigned long mem_end;
1680 unsigned long mem_start;
1681 unsigned long base_addr;
1682 int irq;
1683
1684
1685
1686
1687
1688
1689
1690 unsigned long state;
1691
1692 struct list_head dev_list;
1693 struct list_head napi_list;
1694 struct list_head unreg_list;
1695 struct list_head close_list;
1696 struct list_head ptype_all;
1697 struct list_head ptype_specific;
1698
1699 struct {
1700 struct list_head upper;
1701 struct list_head lower;
1702 } adj_list;
1703
1704 netdev_features_t features;
1705 netdev_features_t hw_features;
1706 netdev_features_t wanted_features;
1707 netdev_features_t vlan_features;
1708 netdev_features_t hw_enc_features;
1709 netdev_features_t mpls_features;
1710 netdev_features_t gso_partial_features;
1711
1712 int ifindex;
1713 int group;
1714
1715 struct net_device_stats stats;
1716
1717 atomic_long_t rx_dropped;
1718 atomic_long_t tx_dropped;
1719 atomic_long_t rx_nohandler;
1720
1721
1722 atomic_t carrier_up_count;
1723 atomic_t carrier_down_count;
1724
1725#ifdef CONFIG_WIRELESS_EXT
1726 const struct iw_handler_def *wireless_handlers;
1727 struct iw_public_data *wireless_data;
1728#endif
1729 const struct net_device_ops *netdev_ops;
1730 const struct ethtool_ops *ethtool_ops;
1731#ifdef CONFIG_NET_SWITCHDEV
1732 const struct switchdev_ops *switchdev_ops;
1733#endif
1734#ifdef CONFIG_NET_L3_MASTER_DEV
1735 const struct l3mdev_ops *l3mdev_ops;
1736#endif
1737#if IS_ENABLED(CONFIG_IPV6)
1738 const struct ndisc_ops *ndisc_ops;
1739#endif
1740
1741#ifdef CONFIG_XFRM_OFFLOAD
1742 const struct xfrmdev_ops *xfrmdev_ops;
1743#endif
1744
1745 const struct header_ops *header_ops;
1746
1747 unsigned int flags;
1748 unsigned int priv_flags;
1749
1750 unsigned short gflags;
1751 unsigned short padded;
1752
1753 unsigned char operstate;
1754 unsigned char link_mode;
1755
1756 unsigned char if_port;
1757 unsigned char dma;
1758
1759 unsigned int mtu;
1760 unsigned int min_mtu;
1761 unsigned int max_mtu;
1762 unsigned short type;
1763 unsigned short hard_header_len;
1764 unsigned char min_header_len;
1765
1766 unsigned short needed_headroom;
1767 unsigned short needed_tailroom;
1768
1769
1770 unsigned char perm_addr[MAX_ADDR_LEN];
1771 unsigned char addr_assign_type;
1772 unsigned char addr_len;
1773 unsigned short neigh_priv_len;
1774 unsigned short dev_id;
1775 unsigned short dev_port;
1776 spinlock_t addr_list_lock;
1777 unsigned char name_assign_type;
1778 bool uc_promisc;
1779 struct netdev_hw_addr_list uc;
1780 struct netdev_hw_addr_list mc;
1781 struct netdev_hw_addr_list dev_addrs;
1782
1783#ifdef CONFIG_SYSFS
1784 struct kset *queues_kset;
1785#endif
1786 unsigned int promiscuity;
1787 unsigned int allmulti;
1788
1789
1790
1791
1792#if IS_ENABLED(CONFIG_VLAN_8021Q)
1793 struct vlan_info __rcu *vlan_info;
1794#endif
1795#if IS_ENABLED(CONFIG_NET_DSA)
1796 struct dsa_port *dsa_ptr;
1797#endif
1798#if IS_ENABLED(CONFIG_TIPC)
1799 struct tipc_bearer __rcu *tipc_ptr;
1800#endif
1801 void *atalk_ptr;
1802 struct in_device __rcu *ip_ptr;
1803 struct dn_dev __rcu *dn_ptr;
1804 struct inet6_dev __rcu *ip6_ptr;
1805 void *ax25_ptr;
1806 struct wireless_dev *ieee80211_ptr;
1807 struct wpan_dev *ieee802154_ptr;
1808#if IS_ENABLED(CONFIG_MPLS_ROUTING)
1809 struct mpls_dev __rcu *mpls_ptr;
1810#endif
1811
1812
1813
1814
1815
1816 unsigned char *dev_addr;
1817
1818 struct netdev_rx_queue *_rx;
1819 unsigned int num_rx_queues;
1820 unsigned int real_num_rx_queues;
1821
1822 struct bpf_prog __rcu *xdp_prog;
1823 unsigned long gro_flush_timeout;
1824 rx_handler_func_t __rcu *rx_handler;
1825 void __rcu *rx_handler_data;
1826
1827#ifdef CONFIG_NET_CLS_ACT
1828 struct mini_Qdisc __rcu *miniq_ingress;
1829#endif
1830 struct netdev_queue __rcu *ingress_queue;
1831#ifdef CONFIG_NETFILTER_INGRESS
1832 struct nf_hook_entries __rcu *nf_hooks_ingress;
1833#endif
1834
1835 unsigned char broadcast[MAX_ADDR_LEN];
1836#ifdef CONFIG_RFS_ACCEL
1837 struct cpu_rmap *rx_cpu_rmap;
1838#endif
1839 struct hlist_node index_hlist;
1840
1841
1842
1843
1844 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
1845 unsigned int num_tx_queues;
1846 unsigned int real_num_tx_queues;
1847 struct Qdisc *qdisc;
1848#ifdef CONFIG_NET_SCHED
1849 DECLARE_HASHTABLE (qdisc_hash, 4);
1850#endif
1851 unsigned int tx_queue_len;
1852 spinlock_t tx_global_lock;
1853 int watchdog_timeo;
1854
1855#ifdef CONFIG_XPS
1856 struct xps_dev_maps __rcu *xps_maps;
1857#endif
1858#ifdef CONFIG_NET_CLS_ACT
1859 struct mini_Qdisc __rcu *miniq_egress;
1860#endif
1861
1862
1863 struct timer_list watchdog_timer;
1864
1865 int __percpu *pcpu_refcnt;
1866 struct list_head todo_list;
1867
1868 struct list_head link_watch_list;
1869
1870 enum { NETREG_UNINITIALIZED=0,
1871 NETREG_REGISTERED,
1872 NETREG_UNREGISTERING,
1873 NETREG_UNREGISTERED,
1874 NETREG_RELEASED,
1875 NETREG_DUMMY,
1876 } reg_state:8;
1877
1878 bool dismantle;
1879
1880 enum {
1881 RTNL_LINK_INITIALIZED,
1882 RTNL_LINK_INITIALIZING,
1883 } rtnl_link_state:16;
1884
1885 bool needs_free_netdev;
1886 void (*priv_destructor)(struct net_device *dev);
1887
1888#ifdef CONFIG_NETPOLL
1889 struct netpoll_info __rcu *npinfo;
1890#endif
1891
1892 possible_net_t nd_net;
1893
1894
1895 union {
1896 void *ml_priv;
1897 struct pcpu_lstats __percpu *lstats;
1898 struct pcpu_sw_netstats __percpu *tstats;
1899 struct pcpu_dstats __percpu *dstats;
1900 struct pcpu_vstats __percpu *vstats;
1901 };
1902
1903#if IS_ENABLED(CONFIG_GARP)
1904 struct garp_port __rcu *garp_port;
1905#endif
1906#if IS_ENABLED(CONFIG_MRP)
1907 struct mrp_port __rcu *mrp_port;
1908#endif
1909
1910 struct device dev;
1911 const struct attribute_group *sysfs_groups[4];
1912 const struct attribute_group *sysfs_rx_queue_group;
1913
1914 const struct rtnl_link_ops *rtnl_link_ops;
1915
1916
1917#define GSO_MAX_SIZE 65536
1918 unsigned int gso_max_size;
1919#define GSO_MAX_SEGS 65535
1920 u16 gso_max_segs;
1921
1922#ifdef CONFIG_DCB
1923 const struct dcbnl_rtnl_ops *dcbnl_ops;
1924#endif
1925 u8 num_tc;
1926 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1927 u8 prio_tc_map[TC_BITMASK + 1];
1928
1929#if IS_ENABLED(CONFIG_FCOE)
1930 unsigned int fcoe_ddp_xid;
1931#endif
1932#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
1933 struct netprio_map __rcu *priomap;
1934#endif
1935 struct phy_device *phydev;
1936 struct lock_class_key *qdisc_tx_busylock;
1937 struct lock_class_key *qdisc_running_key;
1938 bool proto_down;
1939};
1940#define to_net_dev(d) container_of(d, struct net_device, dev)
1941
1942static inline bool netif_elide_gro(const struct net_device *dev)
1943{
1944 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
1945 return true;
1946 return false;
1947}
1948
1949#define NETDEV_ALIGN 32
1950
1951static inline
1952int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1953{
1954 return dev->prio_tc_map[prio & TC_BITMASK];
1955}
1956
1957static inline
1958int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1959{
1960 if (tc >= dev->num_tc)
1961 return -EINVAL;
1962
1963 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
1964 return 0;
1965}
1966
1967int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
1968void netdev_reset_tc(struct net_device *dev);
1969int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
1970int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
1971
1972static inline
1973int netdev_get_num_tc(struct net_device *dev)
1974{
1975 return dev->num_tc;
1976}
1977
1978static inline
1979struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
1980 unsigned int index)
1981{
1982 return &dev->_tx[index];
1983}
1984
1985static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
1986 const struct sk_buff *skb)
1987{
1988 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
1989}
1990
1991static inline void netdev_for_each_tx_queue(struct net_device *dev,
1992 void (*f)(struct net_device *,
1993 struct netdev_queue *,
1994 void *),
1995 void *arg)
1996{
1997 unsigned int i;
1998
1999 for (i = 0; i < dev->num_tx_queues; i++)
2000 f(dev, &dev->_tx[i], arg);
2001}
2002
2003#define netdev_lockdep_set_classes(dev) \
2004{ \
2005 static struct lock_class_key qdisc_tx_busylock_key; \
2006 static struct lock_class_key qdisc_running_key; \
2007 static struct lock_class_key qdisc_xmit_lock_key; \
2008 static struct lock_class_key dev_addr_list_lock_key; \
2009 unsigned int i; \
2010 \
2011 (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \
2012 (dev)->qdisc_running_key = &qdisc_running_key; \
2013 lockdep_set_class(&(dev)->addr_list_lock, \
2014 &dev_addr_list_lock_key); \
2015 for (i = 0; i < (dev)->num_tx_queues; i++) \
2016 lockdep_set_class(&(dev)->_tx[i]._xmit_lock, \
2017 &qdisc_xmit_lock_key); \
2018}
2019
2020struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2021 struct sk_buff *skb,
2022 void *accel_priv);
2023
2024
2025
2026
2027static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2028{
2029 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2030}
2031
2032static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2033{
2034 if (dev->netdev_ops->ndo_set_rx_headroom)
2035 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2036}
2037
2038
2039static inline void netdev_reset_rx_headroom(struct net_device *dev)
2040{
2041 netdev_set_rx_headroom(dev, -1);
2042}
2043
2044
2045
2046
2047static inline
2048struct net *dev_net(const struct net_device *dev)
2049{
2050 return read_pnet(&dev->nd_net);
2051}
2052
2053static inline
2054void dev_net_set(struct net_device *dev, struct net *net)
2055{
2056 write_pnet(&dev->nd_net, net);
2057}
2058
2059
2060
2061
2062
2063
2064
2065static inline void *netdev_priv(const struct net_device *dev)
2066{
2067 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
2068}
2069
2070
2071
2072
2073#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
2074
2075
2076
2077
2078
2079#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2080
2081
2082
2083
2084#define NAPI_POLL_WEIGHT 64
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2097 int (*poll)(struct napi_struct *, int), int weight);
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110static inline void netif_tx_napi_add(struct net_device *dev,
2111 struct napi_struct *napi,
2112 int (*poll)(struct napi_struct *, int),
2113 int weight)
2114{
2115 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2116 netif_napi_add(dev, napi, poll, weight);
2117}
2118
2119
2120
2121
2122
2123
2124
2125void netif_napi_del(struct napi_struct *napi);
2126
2127struct napi_gro_cb {
2128
2129 void *frag0;
2130
2131
2132 unsigned int frag0_len;
2133
2134
2135 int data_offset;
2136
2137
2138 u16 flush;
2139
2140
2141 u16 flush_id;
2142
2143
2144 u16 count;
2145
2146
2147 u16 gro_remcsum_start;
2148
2149
2150 unsigned long age;
2151
2152
2153 u16 proto;
2154
2155
2156 u8 same_flow:1;
2157
2158
2159 u8 encap_mark:1;
2160
2161
2162 u8 csum_valid:1;
2163
2164
2165 u8 csum_cnt:3;
2166
2167
2168 u8 free:2;
2169#define NAPI_GRO_FREE 1
2170#define NAPI_GRO_FREE_STOLEN_HEAD 2
2171
2172
2173 u8 is_ipv6:1;
2174
2175
2176 u8 is_fou:1;
2177
2178
2179 u8 is_atomic:1;
2180
2181
2182 u8 recursion_counter:4;
2183
2184
2185
2186
2187 __wsum csum;
2188
2189
2190 struct sk_buff *last;
2191};
2192
2193#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
2194
2195#define GRO_RECURSION_LIMIT 15
2196static inline int gro_recursion_inc_test(struct sk_buff *skb)
2197{
2198 return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
2199}
2200
2201typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);
2202static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
2203 struct sk_buff **head,
2204 struct sk_buff *skb)
2205{
2206 if (unlikely(gro_recursion_inc_test(skb))) {
2207 NAPI_GRO_CB(skb)->flush |= 1;
2208 return NULL;
2209 }
2210
2211 return cb(head, skb);
2212}
2213
2214typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,
2215 struct sk_buff *);
2216static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,
2217 struct sock *sk,
2218 struct sk_buff **head,
2219 struct sk_buff *skb)
2220{
2221 if (unlikely(gro_recursion_inc_test(skb))) {
2222 NAPI_GRO_CB(skb)->flush |= 1;
2223 return NULL;
2224 }
2225
2226 return cb(sk, head, skb);
2227}
2228
2229struct packet_type {
2230 __be16 type;
2231 struct net_device *dev;
2232 int (*func) (struct sk_buff *,
2233 struct net_device *,
2234 struct packet_type *,
2235 struct net_device *);
2236 bool (*id_match)(struct packet_type *ptype,
2237 struct sock *sk);
2238 void *af_packet_priv;
2239 struct list_head list;
2240};
2241
2242struct offload_callbacks {
2243 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2244 netdev_features_t features);
2245 struct sk_buff **(*gro_receive)(struct sk_buff **head,
2246 struct sk_buff *skb);
2247 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2248};
2249
2250struct packet_offload {
2251 __be16 type;
2252 u16 priority;
2253 struct offload_callbacks callbacks;
2254 struct list_head list;
2255};
2256
2257
2258struct pcpu_sw_netstats {
2259 u64 rx_packets;
2260 u64 rx_bytes;
2261 u64 tx_packets;
2262 u64 tx_bytes;
2263 struct u64_stats_sync syncp;
2264};
2265
2266#define __netdev_alloc_pcpu_stats(type, gfp) \
2267({ \
2268 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2269 if (pcpu_stats) { \
2270 int __cpu; \
2271 for_each_possible_cpu(__cpu) { \
2272 typeof(type) *stat; \
2273 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2274 u64_stats_init(&stat->syncp); \
2275 } \
2276 } \
2277 pcpu_stats; \
2278})
2279
2280#define netdev_alloc_pcpu_stats(type) \
2281 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2282
2283enum netdev_lag_tx_type {
2284 NETDEV_LAG_TX_TYPE_UNKNOWN,
2285 NETDEV_LAG_TX_TYPE_RANDOM,
2286 NETDEV_LAG_TX_TYPE_BROADCAST,
2287 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2288 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2289 NETDEV_LAG_TX_TYPE_HASH,
2290};
2291
2292struct netdev_lag_upper_info {
2293 enum netdev_lag_tx_type tx_type;
2294};
2295
2296struct netdev_lag_lower_state_info {
2297 u8 link_up : 1,
2298 tx_enabled : 1;
2299};
2300
2301#include <linux/notifier.h>
2302
2303
2304
2305
2306
2307#define NETDEV_UP 0x0001
2308#define NETDEV_DOWN 0x0002
2309#define NETDEV_REBOOT 0x0003
2310
2311
2312
2313#define NETDEV_CHANGE 0x0004
2314#define NETDEV_REGISTER 0x0005
2315#define NETDEV_UNREGISTER 0x0006
2316#define NETDEV_CHANGEMTU 0x0007
2317#define NETDEV_CHANGEADDR 0x0008
2318#define NETDEV_GOING_DOWN 0x0009
2319#define NETDEV_CHANGENAME 0x000A
2320#define NETDEV_FEAT_CHANGE 0x000B
2321#define NETDEV_BONDING_FAILOVER 0x000C
2322#define NETDEV_PRE_UP 0x000D
2323#define NETDEV_PRE_TYPE_CHANGE 0x000E
2324#define NETDEV_POST_TYPE_CHANGE 0x000F
2325#define NETDEV_POST_INIT 0x0010
2326#define NETDEV_UNREGISTER_FINAL 0x0011
2327#define NETDEV_RELEASE 0x0012
2328#define NETDEV_NOTIFY_PEERS 0x0013
2329#define NETDEV_JOIN 0x0014
2330#define NETDEV_CHANGEUPPER 0x0015
2331#define NETDEV_RESEND_IGMP 0x0016
2332#define NETDEV_PRECHANGEMTU 0x0017
2333#define NETDEV_CHANGEINFODATA 0x0018
2334#define NETDEV_BONDING_INFO 0x0019
2335#define NETDEV_PRECHANGEUPPER 0x001A
2336#define NETDEV_CHANGELOWERSTATE 0x001B
2337#define NETDEV_UDP_TUNNEL_PUSH_INFO 0x001C
2338#define NETDEV_UDP_TUNNEL_DROP_INFO 0x001D
2339#define NETDEV_CHANGE_TX_QUEUE_LEN 0x001E
2340
2341int register_netdevice_notifier(struct notifier_block *nb);
2342int unregister_netdevice_notifier(struct notifier_block *nb);
2343
2344struct netdev_notifier_info {
2345 struct net_device *dev;
2346 struct netlink_ext_ack *extack;
2347};
2348
2349struct netdev_notifier_change_info {
2350 struct netdev_notifier_info info;
2351 unsigned int flags_changed;
2352};
2353
2354struct netdev_notifier_changeupper_info {
2355 struct netdev_notifier_info info;
2356 struct net_device *upper_dev;
2357 bool master;
2358 bool linking;
2359 void *upper_info;
2360};
2361
2362struct netdev_notifier_changelowerstate_info {
2363 struct netdev_notifier_info info;
2364 void *lower_state_info;
2365};
2366
2367static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2368 struct net_device *dev)
2369{
2370 info->dev = dev;
2371 info->extack = NULL;
2372}
2373
2374static inline struct net_device *
2375netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2376{
2377 return info->dev;
2378}
2379
2380static inline struct netlink_ext_ack *
2381netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2382{
2383 return info->extack;
2384}
2385
2386int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2387
2388
2389extern rwlock_t dev_base_lock;
2390
2391#define for_each_netdev(net, d) \
2392 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2393#define for_each_netdev_reverse(net, d) \
2394 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2395#define for_each_netdev_rcu(net, d) \
2396 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2397#define for_each_netdev_safe(net, d, n) \
2398 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2399#define for_each_netdev_continue(net, d) \
2400 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2401#define for_each_netdev_continue_rcu(net, d) \
2402 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2403#define for_each_netdev_in_bond_rcu(bond, slave) \
2404 for_each_netdev_rcu(&init_net, slave) \
2405 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2406#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
2407
2408static inline struct net_device *next_net_device(struct net_device *dev)
2409{
2410 struct list_head *lh;
2411 struct net *net;
2412
2413 net = dev_net(dev);
2414 lh = dev->dev_list.next;
2415 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2416}
2417
2418static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2419{
2420 struct list_head *lh;
2421 struct net *net;
2422
2423 net = dev_net(dev);
2424 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2425 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2426}
2427
2428static inline struct net_device *first_net_device(struct net *net)
2429{
2430 return list_empty(&net->dev_base_head) ? NULL :
2431 net_device_entry(net->dev_base_head.next);
2432}
2433
2434static inline struct net_device *first_net_device_rcu(struct net *net)
2435{
2436 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2437
2438 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2439}
2440
2441int netdev_boot_setup_check(struct net_device *dev);
2442unsigned long netdev_boot_base(const char *prefix, int unit);
2443struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2444 const char *hwaddr);
2445struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2446struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
2447void dev_add_pack(struct packet_type *pt);
2448void dev_remove_pack(struct packet_type *pt);
2449void __dev_remove_pack(struct packet_type *pt);
2450void dev_add_offload(struct packet_offload *po);
2451void dev_remove_offload(struct packet_offload *po);
2452
2453int dev_get_iflink(const struct net_device *dev);
2454int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
2455struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2456 unsigned short mask);
2457struct net_device *dev_get_by_name(struct net *net, const char *name);
2458struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2459struct net_device *__dev_get_by_name(struct net *net, const char *name);
2460int dev_alloc_name(struct net_device *dev, const char *name);
2461int dev_open(struct net_device *dev);
2462void dev_close(struct net_device *dev);
2463void dev_close_many(struct list_head *head, bool unlink);
2464void dev_disable_lro(struct net_device *dev);
2465int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
2466int dev_queue_xmit(struct sk_buff *skb);
2467int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
2468int register_netdevice(struct net_device *dev);
2469void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
2470void unregister_netdevice_many(struct list_head *head);
2471static inline void unregister_netdevice(struct net_device *dev)
2472{
2473 unregister_netdevice_queue(dev, NULL);
2474}
2475
2476int netdev_refcnt_read(const struct net_device *dev);
2477void free_netdev(struct net_device *dev);
2478void netdev_freemem(struct net_device *dev);
2479void synchronize_net(void);
2480int init_dummy_netdev(struct net_device *dev);
2481
2482DECLARE_PER_CPU(int, xmit_recursion);
2483#define XMIT_RECURSION_LIMIT 10
2484
2485static inline int dev_recursion_level(void)
2486{
2487 return this_cpu_read(xmit_recursion);
2488}
2489
2490struct net_device *dev_get_by_index(struct net *net, int ifindex);
2491struct net_device *__dev_get_by_index(struct net *net, int ifindex);
2492struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
2493struct net_device *dev_get_by_napi_id(unsigned int napi_id);
2494int netdev_get_name(struct net *net, char *name, int ifindex);
2495int dev_restart(struct net_device *dev);
2496int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2497
2498static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
2499{
2500 return NAPI_GRO_CB(skb)->data_offset;
2501}
2502
2503static inline unsigned int skb_gro_len(const struct sk_buff *skb)
2504{
2505 return skb->len - NAPI_GRO_CB(skb)->data_offset;
2506}
2507
2508static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
2509{
2510 NAPI_GRO_CB(skb)->data_offset += len;
2511}
2512
2513static inline void *skb_gro_header_fast(struct sk_buff *skb,
2514 unsigned int offset)
2515{
2516 return NAPI_GRO_CB(skb)->frag0 + offset;
2517}
2518
2519static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
2520{
2521 return NAPI_GRO_CB(skb)->frag0_len < hlen;
2522}
2523
2524static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
2525{
2526 NAPI_GRO_CB(skb)->frag0 = NULL;
2527 NAPI_GRO_CB(skb)->frag0_len = 0;
2528}
2529
2530static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
2531 unsigned int offset)
2532{
2533 if (!pskb_may_pull(skb, hlen))
2534 return NULL;
2535
2536 skb_gro_frag0_invalidate(skb);
2537 return skb->data + offset;
2538}
2539
2540static inline void *skb_gro_network_header(struct sk_buff *skb)
2541{
2542 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
2543 skb_network_offset(skb);
2544}
2545
2546static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
2547 const void *start, unsigned int len)
2548{
2549 if (NAPI_GRO_CB(skb)->csum_valid)
2550 NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
2551 csum_partial(start, len, 0));
2552}
2553
2554
2555
2556
2557
2558
2559__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
2560
2561static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
2562{
2563 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
2564}
2565
2566static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
2567 bool zero_okay,
2568 __sum16 check)
2569{
2570 return ((skb->ip_summed != CHECKSUM_PARTIAL ||
2571 skb_checksum_start_offset(skb) <
2572 skb_gro_offset(skb)) &&
2573 !skb_at_gro_remcsum_start(skb) &&
2574 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2575 (!zero_okay || check));
2576}
2577
2578static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
2579 __wsum psum)
2580{
2581 if (NAPI_GRO_CB(skb)->csum_valid &&
2582 !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
2583 return 0;
2584
2585 NAPI_GRO_CB(skb)->csum = psum;
2586
2587 return __skb_gro_checksum_complete(skb);
2588}
2589
2590static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
2591{
2592 if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
2593
2594 NAPI_GRO_CB(skb)->csum_cnt--;
2595 } else {
2596
2597
2598
2599
2600 __skb_incr_checksum_unnecessary(skb);
2601 }
2602}
2603
2604#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \
2605 compute_pseudo) \
2606({ \
2607 __sum16 __ret = 0; \
2608 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \
2609 __ret = __skb_gro_checksum_validate_complete(skb, \
2610 compute_pseudo(skb, proto)); \
2611 if (!__ret) \
2612 skb_gro_incr_csum_unnecessary(skb); \
2613 __ret; \
2614})
2615
2616#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \
2617 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
2618
2619#define skb_gro_checksum_validate_zero_check(skb, proto, check, \
2620 compute_pseudo) \
2621 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
2622
2623#define skb_gro_checksum_simple_validate(skb) \
2624 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
2625
2626static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
2627{
2628 return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2629 !NAPI_GRO_CB(skb)->csum_valid);
2630}
2631
2632static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2633 __sum16 check, __wsum pseudo)
2634{
2635 NAPI_GRO_CB(skb)->csum = ~pseudo;
2636 NAPI_GRO_CB(skb)->csum_valid = 1;
2637}
2638
2639#define skb_gro_checksum_try_convert(skb, proto, check, compute_pseudo) \
2640do { \
2641 if (__skb_gro_checksum_convert_check(skb)) \
2642 __skb_gro_checksum_convert(skb, check, \
2643 compute_pseudo(skb, proto)); \
2644} while (0)
2645
2646struct gro_remcsum {
2647 int offset;
2648 __wsum delta;
2649};
2650
2651static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
2652{
2653 grc->offset = 0;
2654 grc->delta = 0;
2655}
2656
2657static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
2658 unsigned int off, size_t hdrlen,
2659 int start, int offset,
2660 struct gro_remcsum *grc,
2661 bool nopartial)
2662{
2663 __wsum delta;
2664 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
2665
2666 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
2667
2668 if (!nopartial) {
2669 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
2670 return ptr;
2671 }
2672
2673 ptr = skb_gro_header_fast(skb, off);
2674 if (skb_gro_header_hard(skb, off + plen)) {
2675 ptr = skb_gro_header_slow(skb, off + plen, off);
2676 if (!ptr)
2677 return NULL;
2678 }
2679
2680 delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
2681 start, offset);
2682
2683
2684 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
2685
2686 grc->offset = off + hdrlen + offset;
2687 grc->delta = delta;
2688
2689 return ptr;
2690}
2691
2692static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
2693 struct gro_remcsum *grc)
2694{
2695 void *ptr;
2696 size_t plen = grc->offset + sizeof(u16);
2697
2698 if (!grc->delta)
2699 return;
2700
2701 ptr = skb_gro_header_fast(skb, grc->offset);
2702 if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
2703 ptr = skb_gro_header_slow(skb, plen, grc->offset);
2704 if (!ptr)
2705 return;
2706 }
2707
2708 remcsum_unadjust((__sum16 *)ptr, grc->delta);
2709}
2710
2711#ifdef CONFIG_XFRM_OFFLOAD
2712static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2713{
2714 if (PTR_ERR(pp) != -EINPROGRESS)
2715 NAPI_GRO_CB(skb)->flush |= flush;
2716}
2717#else
2718static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2719{
2720 NAPI_GRO_CB(skb)->flush |= flush;
2721}
2722#endif
2723
2724static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
2725 unsigned short type,
2726 const void *daddr, const void *saddr,
2727 unsigned int len)
2728{
2729 if (!dev->header_ops || !dev->header_ops->create)
2730 return 0;
2731
2732 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
2733}
2734
2735static inline int dev_parse_header(const struct sk_buff *skb,
2736 unsigned char *haddr)
2737{
2738 const struct net_device *dev = skb->dev;
2739
2740 if (!dev->header_ops || !dev->header_ops->parse)
2741 return 0;
2742 return dev->header_ops->parse(skb, haddr);
2743}
2744
2745
2746static inline bool dev_validate_header(const struct net_device *dev,
2747 char *ll_header, int len)
2748{
2749 if (likely(len >= dev->hard_header_len))
2750 return true;
2751 if (len < dev->min_header_len)
2752 return false;
2753
2754 if (capable(CAP_SYS_RAWIO)) {
2755 memset(ll_header + len, 0, dev->hard_header_len - len);
2756 return true;
2757 }
2758
2759 if (dev->header_ops && dev->header_ops->validate)
2760 return dev->header_ops->validate(ll_header, len);
2761
2762 return false;
2763}
2764
2765typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr,
2766 int len, int size);
2767int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
2768static inline int unregister_gifconf(unsigned int family)
2769{
2770 return register_gifconf(family, NULL);
2771}
2772
2773#ifdef CONFIG_NET_FLOW_LIMIT
2774#define FLOW_LIMIT_HISTORY (1 << 7)
2775struct sd_flow_limit {
2776 u64 count;
2777 unsigned int num_buckets;
2778 unsigned int history_head;
2779 u16 history[FLOW_LIMIT_HISTORY];
2780 u8 buckets[];
2781};
2782
2783extern int netdev_flow_limit_table_len;
2784#endif
2785
2786
2787
2788
2789struct softnet_data {
2790 struct list_head poll_list;
2791 struct sk_buff_head process_queue;
2792
2793
2794 unsigned int processed;
2795 unsigned int time_squeeze;
2796 unsigned int received_rps;
2797#ifdef CONFIG_RPS
2798 struct softnet_data *rps_ipi_list;
2799#endif
2800#ifdef CONFIG_NET_FLOW_LIMIT
2801 struct sd_flow_limit __rcu *flow_limit;
2802#endif
2803 struct Qdisc *output_queue;
2804 struct Qdisc **output_queue_tailp;
2805 struct sk_buff *completion_queue;
2806#ifdef CONFIG_XFRM_OFFLOAD
2807 struct sk_buff_head xfrm_backlog;
2808#endif
2809#ifdef CONFIG_RPS
2810
2811
2812
2813 unsigned int input_queue_head ____cacheline_aligned_in_smp;
2814
2815
2816 call_single_data_t csd ____cacheline_aligned_in_smp;
2817 struct softnet_data *rps_ipi_next;
2818 unsigned int cpu;
2819 unsigned int input_queue_tail;
2820#endif
2821 unsigned int dropped;
2822 struct sk_buff_head input_pkt_queue;
2823 struct napi_struct backlog;
2824
2825};
2826
2827static inline void input_queue_head_incr(struct softnet_data *sd)
2828{
2829#ifdef CONFIG_RPS
2830 sd->input_queue_head++;
2831#endif
2832}
2833
2834static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2835 unsigned int *qtail)
2836{
2837#ifdef CONFIG_RPS
2838 *qtail = ++sd->input_queue_tail;
2839#endif
2840}
2841
2842DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
2843
2844void __netif_schedule(struct Qdisc *q);
2845void netif_schedule_queue(struct netdev_queue *txq);
2846
2847static inline void netif_tx_schedule_all(struct net_device *dev)
2848{
2849 unsigned int i;
2850
2851 for (i = 0; i < dev->num_tx_queues; i++)
2852 netif_schedule_queue(netdev_get_tx_queue(dev, i));
2853}
2854
2855static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
2856{
2857 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2858}
2859
2860
2861
2862
2863
2864
2865
2866static inline void netif_start_queue(struct net_device *dev)
2867{
2868 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
2869}
2870
2871static inline void netif_tx_start_all_queues(struct net_device *dev)
2872{
2873 unsigned int i;
2874
2875 for (i = 0; i < dev->num_tx_queues; i++) {
2876 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2877 netif_tx_start_queue(txq);
2878 }
2879}
2880
2881void netif_tx_wake_queue(struct netdev_queue *dev_queue);
2882
2883
2884
2885
2886
2887
2888
2889
2890static inline void netif_wake_queue(struct net_device *dev)
2891{
2892 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
2893}
2894
2895static inline void netif_tx_wake_all_queues(struct net_device *dev)
2896{
2897 unsigned int i;
2898
2899 for (i = 0; i < dev->num_tx_queues; i++) {
2900 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2901 netif_tx_wake_queue(txq);
2902 }
2903}
2904
2905static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
2906{
2907 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2908}
2909
2910
2911
2912
2913
2914
2915
2916
2917static inline void netif_stop_queue(struct net_device *dev)
2918{
2919 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
2920}
2921
2922void netif_tx_stop_all_queues(struct net_device *dev);
2923
2924static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
2925{
2926 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2927}
2928
2929
2930
2931
2932
2933
2934
2935static inline bool netif_queue_stopped(const struct net_device *dev)
2936{
2937 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
2938}
2939
2940static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
2941{
2942 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2943}
2944
2945static inline bool
2946netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
2947{
2948 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2949}
2950
2951static inline bool
2952netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
2953{
2954 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
2955}
2956
2957
2958
2959
2960
2961
2962
2963
2964static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
2965{
2966#ifdef CONFIG_BQL
2967 prefetchw(&dev_queue->dql.num_queued);
2968#endif
2969}
2970
2971
2972
2973
2974
2975
2976
2977
2978static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
2979{
2980#ifdef CONFIG_BQL
2981 prefetchw(&dev_queue->dql.limit);
2982#endif
2983}
2984
2985static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
2986 unsigned int bytes)
2987{
2988#ifdef CONFIG_BQL
2989 dql_queued(&dev_queue->dql, bytes);
2990
2991 if (likely(dql_avail(&dev_queue->dql) >= 0))
2992 return;
2993
2994 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2995
2996
2997
2998
2999
3000
3001 smp_mb();
3002
3003
3004 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3005 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3006#endif
3007}
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3019{
3020 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3021}
3022
3023static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3024 unsigned int pkts, unsigned int bytes)
3025{
3026#ifdef CONFIG_BQL
3027 if (unlikely(!bytes))
3028 return;
3029
3030 dql_completed(&dev_queue->dql, bytes);
3031
3032
3033
3034
3035
3036
3037 smp_mb();
3038
3039 if (dql_avail(&dev_queue->dql) < 0)
3040 return;
3041
3042 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3043 netif_schedule_queue(dev_queue);
3044#endif
3045}
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057static inline void netdev_completed_queue(struct net_device *dev,
3058 unsigned int pkts, unsigned int bytes)
3059{
3060 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3061}
3062
3063static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3064{
3065#ifdef CONFIG_BQL
3066 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3067 dql_reset(&q->dql);
3068#endif
3069}
3070
3071
3072
3073
3074
3075
3076
3077
3078static inline void netdev_reset_queue(struct net_device *dev_queue)
3079{
3080 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
3081}
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3092{
3093 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3094 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3095 dev->name, queue_index,
3096 dev->real_num_tx_queues);
3097 return 0;
3098 }
3099
3100 return queue_index;
3101}
3102
3103
3104
3105
3106
3107
3108
3109static inline bool netif_running(const struct net_device *dev)
3110{
3111 return test_bit(__LINK_STATE_START, &dev->state);
3112}
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3129{
3130 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3131
3132 netif_tx_start_queue(txq);
3133}
3134
3135
3136
3137
3138
3139
3140
3141
3142static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3143{
3144 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3145 netif_tx_stop_queue(txq);
3146}
3147
3148
3149
3150
3151
3152
3153
3154
3155static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3156 u16 queue_index)
3157{
3158 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3159
3160 return netif_tx_queue_stopped(txq);
3161}
3162
3163static inline bool netif_subqueue_stopped(const struct net_device *dev,
3164 struct sk_buff *skb)
3165{
3166 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3167}
3168
3169
3170
3171
3172
3173
3174
3175
3176static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3177{
3178 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3179
3180 netif_tx_wake_queue(txq);
3181}
3182
3183#ifdef CONFIG_XPS
3184int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3185 u16 index);
3186#else
3187static inline int netif_set_xps_queue(struct net_device *dev,
3188 const struct cpumask *mask,
3189 u16 index)
3190{
3191 return 0;
3192}
3193#endif
3194
3195u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
3196 unsigned int num_tx_queues);
3197
3198
3199
3200
3201
3202static inline u16 skb_tx_hash(const struct net_device *dev,
3203 struct sk_buff *skb)
3204{
3205 return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
3206}
3207
3208
3209
3210
3211
3212
3213
3214static inline bool netif_is_multiqueue(const struct net_device *dev)
3215{
3216 return dev->num_tx_queues > 1;
3217}
3218
3219int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3220
3221#ifdef CONFIG_SYSFS
3222int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3223#else
3224static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3225 unsigned int rxq)
3226{
3227 return 0;
3228}
3229#endif
3230
3231static inline struct netdev_rx_queue *
3232__netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
3233{
3234 return dev->_rx + rxq;
3235}
3236
3237#ifdef CONFIG_SYSFS
3238static inline unsigned int get_netdev_rx_queue_index(
3239 struct netdev_rx_queue *queue)
3240{
3241 struct net_device *dev = queue->dev;
3242 int index = queue - dev->_rx;
3243
3244 BUG_ON(index >= dev->num_rx_queues);
3245 return index;
3246}
3247#endif
3248
3249#define DEFAULT_MAX_NUM_RSS_QUEUES (8)
3250int netif_get_num_default_rss_queues(void);
3251
3252enum skb_free_reason {
3253 SKB_REASON_CONSUMED,
3254 SKB_REASON_DROPPED,
3255};
3256
3257void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
3258void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3280{
3281 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
3282}
3283
3284static inline void dev_consume_skb_irq(struct sk_buff *skb)
3285{
3286 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
3287}
3288
3289static inline void dev_kfree_skb_any(struct sk_buff *skb)
3290{
3291 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
3292}
3293
3294static inline void dev_consume_skb_any(struct sk_buff *skb)
3295{
3296 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3297}
3298
3299void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3300int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
3301int netif_rx(struct sk_buff *skb);
3302int netif_rx_ni(struct sk_buff *skb);
3303int netif_receive_skb(struct sk_buff *skb);
3304int netif_receive_skb_core(struct sk_buff *skb);
3305gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3306void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3307struct sk_buff *napi_get_frags(struct napi_struct *napi);
3308gro_result_t napi_gro_frags(struct napi_struct *napi);
3309struct packet_offload *gro_find_receive_by_type(__be16 type);
3310struct packet_offload *gro_find_complete_by_type(__be16 type);
3311
3312static inline void napi_free_frags(struct napi_struct *napi)
3313{
3314 kfree_skb(napi->skb);
3315 napi->skb = NULL;
3316}
3317
3318bool netdev_is_rx_handler_busy(struct net_device *dev);
3319int netdev_rx_handler_register(struct net_device *dev,
3320 rx_handler_func_t *rx_handler,
3321 void *rx_handler_data);
3322void netdev_rx_handler_unregister(struct net_device *dev);
3323
3324bool dev_valid_name(const char *name);
3325int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3326 bool *need_copyout);
3327int dev_ifconf(struct net *net, struct ifconf *, int);
3328int dev_ethtool(struct net *net, struct ifreq *);
3329unsigned int dev_get_flags(const struct net_device *);
3330int __dev_change_flags(struct net_device *, unsigned int flags);
3331int dev_change_flags(struct net_device *, unsigned int);
3332void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3333 unsigned int gchanges);
3334int dev_change_name(struct net_device *, const char *);
3335int dev_set_alias(struct net_device *, const char *, size_t);
3336int dev_get_alias(const struct net_device *, char *, size_t);
3337int dev_change_net_namespace(struct net_device *, struct net *, const char *);
3338int __dev_set_mtu(struct net_device *, int);
3339int dev_set_mtu(struct net_device *, int);
3340int dev_change_tx_queue_len(struct net_device *, unsigned long);
3341void dev_set_group(struct net_device *, int);
3342int dev_set_mac_address(struct net_device *, struct sockaddr *);
3343int dev_change_carrier(struct net_device *, bool new_carrier);
3344int dev_get_phys_port_id(struct net_device *dev,
3345 struct netdev_phys_item_id *ppid);
3346int dev_get_phys_port_name(struct net_device *dev,
3347 char *name, size_t len);
3348int dev_change_proto_down(struct net_device *dev, bool proto_down);
3349struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3350struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3351 struct netdev_queue *txq, int *ret);
3352
3353typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
3354int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
3355 int fd, u32 flags);
3356void __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,
3357 struct netdev_bpf *xdp);
3358
3359int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3360int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3361bool is_skb_forwardable(const struct net_device *dev,
3362 const struct sk_buff *skb);
3363
3364static __always_inline int ____dev_forward_skb(struct net_device *dev,
3365 struct sk_buff *skb)
3366{
3367 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3368 unlikely(!is_skb_forwardable(dev, skb))) {
3369 atomic_long_inc(&dev->rx_dropped);
3370 kfree_skb(skb);
3371 return NET_RX_DROP;
3372 }
3373
3374 skb_scrub_packet(skb, true);
3375 skb->priority = 0;
3376 return 0;
3377}
3378
3379void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
3380
3381extern int netdev_budget;
3382extern unsigned int netdev_budget_usecs;
3383
3384
3385void netdev_run_todo(void);
3386
3387
3388
3389
3390
3391
3392
3393static inline void dev_put(struct net_device *dev)
3394{
3395 this_cpu_dec(*dev->pcpu_refcnt);
3396}
3397
3398
3399
3400
3401
3402
3403
3404static inline void dev_hold(struct net_device *dev)
3405{
3406 this_cpu_inc(*dev->pcpu_refcnt);
3407}
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418void linkwatch_init_dev(struct net_device *dev);
3419void linkwatch_fire_event(struct net_device *dev);
3420void linkwatch_forget_dev(struct net_device *dev);
3421
3422
3423
3424
3425
3426
3427
3428static inline bool netif_carrier_ok(const struct net_device *dev)
3429{
3430 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
3431}
3432
3433unsigned long dev_trans_start(struct net_device *dev);
3434
3435void __netdev_watchdog_up(struct net_device *dev);
3436
3437void netif_carrier_on(struct net_device *dev);
3438
3439void netif_carrier_off(struct net_device *dev);
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453static inline void netif_dormant_on(struct net_device *dev)
3454{
3455 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
3456 linkwatch_fire_event(dev);
3457}
3458
3459
3460
3461
3462
3463
3464
3465static inline void netif_dormant_off(struct net_device *dev)
3466{
3467 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
3468 linkwatch_fire_event(dev);
3469}
3470
3471
3472
3473
3474
3475
3476
3477static inline bool netif_dormant(const struct net_device *dev)
3478{
3479 return test_bit(__LINK_STATE_DORMANT, &dev->state);
3480}
3481
3482
3483
3484
3485
3486
3487
3488
3489static inline bool netif_oper_up(const struct net_device *dev)
3490{
3491 return (dev->operstate == IF_OPER_UP ||
3492 dev->operstate == IF_OPER_UNKNOWN );
3493}
3494
3495
3496
3497
3498
3499
3500
3501static inline bool netif_device_present(struct net_device *dev)
3502{
3503 return test_bit(__LINK_STATE_PRESENT, &dev->state);
3504}
3505
3506void netif_device_detach(struct net_device *dev);
3507
3508void netif_device_attach(struct net_device *dev);
3509
3510
3511
3512
3513
3514enum {
3515 NETIF_MSG_DRV = 0x0001,
3516 NETIF_MSG_PROBE = 0x0002,
3517 NETIF_MSG_LINK = 0x0004,
3518 NETIF_MSG_TIMER = 0x0008,
3519 NETIF_MSG_IFDOWN = 0x0010,
3520 NETIF_MSG_IFUP = 0x0020,
3521 NETIF_MSG_RX_ERR = 0x0040,
3522 NETIF_MSG_TX_ERR = 0x0080,
3523 NETIF_MSG_TX_QUEUED = 0x0100,
3524 NETIF_MSG_INTR = 0x0200,
3525 NETIF_MSG_TX_DONE = 0x0400,
3526 NETIF_MSG_RX_STATUS = 0x0800,
3527 NETIF_MSG_PKTDATA = 0x1000,
3528 NETIF_MSG_HW = 0x2000,
3529 NETIF_MSG_WOL = 0x4000,
3530};
3531
3532#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
3533#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
3534#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
3535#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
3536#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
3537#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
3538#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
3539#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
3540#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
3541#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
3542#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
3543#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
3544#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
3545#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
3546#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
3547
3548static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
3549{
3550
3551 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
3552 return default_msg_enable_bits;
3553 if (debug_value == 0)
3554 return 0;
3555
3556 return (1 << debug_value) - 1;
3557}
3558
3559static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
3560{
3561 spin_lock(&txq->_xmit_lock);
3562 txq->xmit_lock_owner = cpu;
3563}
3564
3565static inline bool __netif_tx_acquire(struct netdev_queue *txq)
3566{
3567 __acquire(&txq->_xmit_lock);
3568 return true;
3569}
3570
3571static inline void __netif_tx_release(struct netdev_queue *txq)
3572{
3573 __release(&txq->_xmit_lock);
3574}
3575
3576static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
3577{
3578 spin_lock_bh(&txq->_xmit_lock);
3579 txq->xmit_lock_owner = smp_processor_id();
3580}
3581
3582static inline bool __netif_tx_trylock(struct netdev_queue *txq)
3583{
3584 bool ok = spin_trylock(&txq->_xmit_lock);
3585 if (likely(ok))
3586 txq->xmit_lock_owner = smp_processor_id();
3587 return ok;
3588}
3589
3590static inline void __netif_tx_unlock(struct netdev_queue *txq)
3591{
3592 txq->xmit_lock_owner = -1;
3593 spin_unlock(&txq->_xmit_lock);
3594}
3595
3596static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
3597{
3598 txq->xmit_lock_owner = -1;
3599 spin_unlock_bh(&txq->_xmit_lock);
3600}
3601
3602static inline void txq_trans_update(struct netdev_queue *txq)
3603{
3604 if (txq->xmit_lock_owner != -1)
3605 txq->trans_start = jiffies;
3606}
3607
3608
3609static inline void netif_trans_update(struct net_device *dev)
3610{
3611 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
3612
3613 if (txq->trans_start != jiffies)
3614 txq->trans_start = jiffies;
3615}
3616
3617
3618
3619
3620
3621
3622
3623static inline void netif_tx_lock(struct net_device *dev)
3624{
3625 unsigned int i;
3626 int cpu;
3627
3628 spin_lock(&dev->tx_global_lock);
3629 cpu = smp_processor_id();
3630 for (i = 0; i < dev->num_tx_queues; i++) {
3631 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3632
3633
3634
3635
3636
3637
3638
3639 __netif_tx_lock(txq, cpu);
3640 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
3641 __netif_tx_unlock(txq);
3642 }
3643}
3644
3645static inline void netif_tx_lock_bh(struct net_device *dev)
3646{
3647 local_bh_disable();
3648 netif_tx_lock(dev);
3649}
3650
3651static inline void netif_tx_unlock(struct net_device *dev)
3652{
3653 unsigned int i;
3654
3655 for (i = 0; i < dev->num_tx_queues; i++) {
3656 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3657
3658
3659
3660
3661
3662 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
3663 netif_schedule_queue(txq);
3664 }
3665 spin_unlock(&dev->tx_global_lock);
3666}
3667
3668static inline void netif_tx_unlock_bh(struct net_device *dev)
3669{
3670 netif_tx_unlock(dev);
3671 local_bh_enable();
3672}
3673
3674#define HARD_TX_LOCK(dev, txq, cpu) { \
3675 if ((dev->features & NETIF_F_LLTX) == 0) { \
3676 __netif_tx_lock(txq, cpu); \
3677 } else { \
3678 __netif_tx_acquire(txq); \
3679 } \
3680}
3681
3682#define HARD_TX_TRYLOCK(dev, txq) \
3683 (((dev->features & NETIF_F_LLTX) == 0) ? \
3684 __netif_tx_trylock(txq) : \
3685 __netif_tx_acquire(txq))
3686
3687#define HARD_TX_UNLOCK(dev, txq) { \
3688 if ((dev->features & NETIF_F_LLTX) == 0) { \
3689 __netif_tx_unlock(txq); \
3690 } else { \
3691 __netif_tx_release(txq); \
3692 } \
3693}
3694
3695static inline void netif_tx_disable(struct net_device *dev)
3696{
3697 unsigned int i;
3698 int cpu;
3699
3700 local_bh_disable();
3701 cpu = smp_processor_id();
3702 for (i = 0; i < dev->num_tx_queues; i++) {
3703 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3704
3705 __netif_tx_lock(txq, cpu);
3706 netif_tx_stop_queue(txq);
3707 __netif_tx_unlock(txq);
3708 }
3709 local_bh_enable();
3710}
3711
3712static inline void netif_addr_lock(struct net_device *dev)
3713{
3714 spin_lock(&dev->addr_list_lock);
3715}
3716
3717static inline void netif_addr_lock_nested(struct net_device *dev)
3718{
3719 int subclass = SINGLE_DEPTH_NESTING;
3720
3721 if (dev->netdev_ops->ndo_get_lock_subclass)
3722 subclass = dev->netdev_ops->ndo_get_lock_subclass(dev);
3723
3724 spin_lock_nested(&dev->addr_list_lock, subclass);
3725}
3726
3727static inline void netif_addr_lock_bh(struct net_device *dev)
3728{
3729 spin_lock_bh(&dev->addr_list_lock);
3730}
3731
3732static inline void netif_addr_unlock(struct net_device *dev)
3733{
3734 spin_unlock(&dev->addr_list_lock);
3735}
3736
3737static inline void netif_addr_unlock_bh(struct net_device *dev)
3738{
3739 spin_unlock_bh(&dev->addr_list_lock);
3740}
3741
3742
3743
3744
3745
3746#define for_each_dev_addr(dev, ha) \
3747 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
3748
3749
3750
3751void ether_setup(struct net_device *dev);
3752
3753
3754struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
3755 unsigned char name_assign_type,
3756 void (*setup)(struct net_device *),
3757 unsigned int txqs, unsigned int rxqs);
3758int dev_get_valid_name(struct net *net, struct net_device *dev,
3759 const char *name);
3760
3761#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
3762 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
3763
3764#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
3765 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
3766 count)
3767
3768int register_netdev(struct net_device *dev);
3769void unregister_netdev(struct net_device *dev);
3770
3771
3772int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
3773 struct netdev_hw_addr_list *from_list, int addr_len);
3774void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
3775 struct netdev_hw_addr_list *from_list, int addr_len);
3776int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
3777 struct net_device *dev,
3778 int (*sync)(struct net_device *, const unsigned char *),
3779 int (*unsync)(struct net_device *,
3780 const unsigned char *));
3781void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
3782 struct net_device *dev,
3783 int (*unsync)(struct net_device *,
3784 const unsigned char *));
3785void __hw_addr_init(struct netdev_hw_addr_list *list);
3786
3787
3788int dev_addr_add(struct net_device *dev, const unsigned char *addr,
3789 unsigned char addr_type);
3790int dev_addr_del(struct net_device *dev, const unsigned char *addr,
3791 unsigned char addr_type);
3792void dev_addr_flush(struct net_device *dev);
3793int dev_addr_init(struct net_device *dev);
3794
3795
3796int dev_uc_add(struct net_device *dev, const unsigned char *addr);
3797int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
3798int dev_uc_del(struct net_device *dev, const unsigned char *addr);
3799int dev_uc_sync(struct net_device *to, struct net_device *from);
3800int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
3801void dev_uc_unsync(struct net_device *to, struct net_device *from);
3802void dev_uc_flush(struct net_device *dev);
3803void dev_uc_init(struct net_device *dev);
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814static inline int __dev_uc_sync(struct net_device *dev,
3815 int (*sync)(struct net_device *,
3816 const unsigned char *),
3817 int (*unsync)(struct net_device *,
3818 const unsigned char *))
3819{
3820 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
3821}
3822
3823
3824
3825
3826
3827
3828
3829
3830static inline void __dev_uc_unsync(struct net_device *dev,
3831 int (*unsync)(struct net_device *,
3832 const unsigned char *))
3833{
3834 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
3835}
3836
3837
3838int dev_mc_add(struct net_device *dev, const unsigned char *addr);
3839int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
3840int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
3841int dev_mc_del(struct net_device *dev, const unsigned char *addr);
3842int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
3843int dev_mc_sync(struct net_device *to, struct net_device *from);
3844int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
3845void dev_mc_unsync(struct net_device *to, struct net_device *from);
3846void dev_mc_flush(struct net_device *dev);
3847void dev_mc_init(struct net_device *dev);
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858static inline int __dev_mc_sync(struct net_device *dev,
3859 int (*sync)(struct net_device *,
3860 const unsigned char *),
3861 int (*unsync)(struct net_device *,
3862 const unsigned char *))
3863{
3864 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
3865}
3866
3867
3868
3869
3870
3871
3872
3873
3874static inline void __dev_mc_unsync(struct net_device *dev,
3875 int (*unsync)(struct net_device *,
3876 const unsigned char *))
3877{
3878 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
3879}
3880
3881
3882void dev_set_rx_mode(struct net_device *dev);
3883void __dev_set_rx_mode(struct net_device *dev);
3884int dev_set_promiscuity(struct net_device *dev, int inc);
3885int dev_set_allmulti(struct net_device *dev, int inc);
3886void netdev_state_change(struct net_device *dev);
3887void netdev_notify_peers(struct net_device *dev);
3888void netdev_features_change(struct net_device *dev);
3889
3890void dev_load(struct net *net, const char *name);
3891struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
3892 struct rtnl_link_stats64 *storage);
3893void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
3894 const struct net_device_stats *netdev_stats);
3895
3896extern int netdev_max_backlog;
3897extern int netdev_tstamp_prequeue;
3898extern int weight_p;
3899extern int dev_weight_rx_bias;
3900extern int dev_weight_tx_bias;
3901extern int dev_rx_weight;
3902extern int dev_tx_weight;
3903
3904bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
3905struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
3906 struct list_head **iter);
3907struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
3908 struct list_head **iter);
3909
3910
3911#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
3912 for (iter = &(dev)->adj_list.upper, \
3913 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
3914 updev; \
3915 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
3916
3917int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
3918 int (*fn)(struct net_device *upper_dev,
3919 void *data),
3920 void *data);
3921
3922bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
3923 struct net_device *upper_dev);
3924
3925bool netdev_has_any_upper_dev(struct net_device *dev);
3926
3927void *netdev_lower_get_next_private(struct net_device *dev,
3928 struct list_head **iter);
3929void *netdev_lower_get_next_private_rcu(struct net_device *dev,
3930 struct list_head **iter);
3931
3932#define netdev_for_each_lower_private(dev, priv, iter) \
3933 for (iter = (dev)->adj_list.lower.next, \
3934 priv = netdev_lower_get_next_private(dev, &(iter)); \
3935 priv; \
3936 priv = netdev_lower_get_next_private(dev, &(iter)))
3937
3938#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
3939 for (iter = &(dev)->adj_list.lower, \
3940 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
3941 priv; \
3942 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
3943
3944void *netdev_lower_get_next(struct net_device *dev,
3945 struct list_head **iter);
3946
3947#define netdev_for_each_lower_dev(dev, ldev, iter) \
3948 for (iter = (dev)->adj_list.lower.next, \
3949 ldev = netdev_lower_get_next(dev, &(iter)); \
3950 ldev; \
3951 ldev = netdev_lower_get_next(dev, &(iter)))
3952
3953struct net_device *netdev_all_lower_get_next(struct net_device *dev,
3954 struct list_head **iter);
3955struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev,
3956 struct list_head **iter);
3957
3958int netdev_walk_all_lower_dev(struct net_device *dev,
3959 int (*fn)(struct net_device *lower_dev,
3960 void *data),
3961 void *data);
3962int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
3963 int (*fn)(struct net_device *lower_dev,
3964 void *data),
3965 void *data);
3966
3967void *netdev_adjacent_get_private(struct list_head *adj_list);
3968void *netdev_lower_get_first_private_rcu(struct net_device *dev);
3969struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
3970struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
3971int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
3972 struct netlink_ext_ack *extack);
3973int netdev_master_upper_dev_link(struct net_device *dev,
3974 struct net_device *upper_dev,
3975 void *upper_priv, void *upper_info,
3976 struct netlink_ext_ack *extack);
3977void netdev_upper_dev_unlink(struct net_device *dev,
3978 struct net_device *upper_dev);
3979void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
3980void *netdev_lower_dev_get_private(struct net_device *dev,
3981 struct net_device *lower_dev);
3982void netdev_lower_state_changed(struct net_device *lower_dev,
3983 void *lower_state_info);
3984
3985
3986#define NETDEV_RSS_KEY_LEN 52
3987extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
3988void netdev_rss_key_fill(void *buffer, size_t len);
3989
3990int dev_get_nest_level(struct net_device *dev);
3991int skb_checksum_help(struct sk_buff *skb);
3992int skb_crc32c_csum_help(struct sk_buff *skb);
3993int skb_csum_hwoffload_help(struct sk_buff *skb,
3994 const netdev_features_t features);
3995
3996struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3997 netdev_features_t features, bool tx_path);
3998struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3999 netdev_features_t features);
4000
4001struct netdev_bonding_info {
4002 ifslave slave;
4003 ifbond master;
4004};
4005
4006struct netdev_notifier_bonding_info {
4007 struct netdev_notifier_info info;
4008 struct netdev_bonding_info bonding_info;
4009};
4010
4011void netdev_bonding_info_change(struct net_device *dev,
4012 struct netdev_bonding_info *bonding_info);
4013
4014static inline
4015struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
4016{
4017 return __skb_gso_segment(skb, features, true);
4018}
4019__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4020
4021static inline bool can_checksum_protocol(netdev_features_t features,
4022 __be16 protocol)
4023{
4024 if (protocol == htons(ETH_P_FCOE))
4025 return !!(features & NETIF_F_FCOE_CRC);
4026
4027
4028
4029 if (features & NETIF_F_HW_CSUM) {
4030
4031 return true;
4032 }
4033
4034 switch (protocol) {
4035 case htons(ETH_P_IP):
4036 return !!(features & NETIF_F_IP_CSUM);
4037 case htons(ETH_P_IPV6):
4038 return !!(features & NETIF_F_IPV6_CSUM);
4039 default:
4040 return false;
4041 }
4042}
4043
4044#ifdef CONFIG_BUG
4045void netdev_rx_csum_fault(struct net_device *dev);
4046#else
4047static inline void netdev_rx_csum_fault(struct net_device *dev)
4048{
4049}
4050#endif
4051
4052void net_enable_timestamp(void);
4053void net_disable_timestamp(void);
4054
4055#ifdef CONFIG_PROC_FS
4056int __init dev_proc_init(void);
4057#else
4058#define dev_proc_init() 0
4059#endif
4060
4061static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4062 struct sk_buff *skb, struct net_device *dev,
4063 bool more)
4064{
4065 skb->xmit_more = more ? 1 : 0;
4066 return ops->ndo_start_xmit(skb, dev);
4067}
4068
4069static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4070 struct netdev_queue *txq, bool more)
4071{
4072 const struct net_device_ops *ops = dev->netdev_ops;
4073 int rc;
4074
4075 rc = __netdev_start_xmit(ops, skb, dev, more);
4076 if (rc == NETDEV_TX_OK)
4077 txq_trans_update(txq);
4078
4079 return rc;
4080}
4081
4082int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4083 const void *ns);
4084void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4085 const void *ns);
4086
4087static inline int netdev_class_create_file(const struct class_attribute *class_attr)
4088{
4089 return netdev_class_create_file_ns(class_attr, NULL);
4090}
4091
4092static inline void netdev_class_remove_file(const struct class_attribute *class_attr)
4093{
4094 netdev_class_remove_file_ns(class_attr, NULL);
4095}
4096
4097extern const struct kobj_ns_type_operations net_ns_type_operations;
4098
4099const char *netdev_drivername(const struct net_device *dev);
4100
4101void linkwatch_run_queue(void);
4102
4103static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4104 netdev_features_t f2)
4105{
4106 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4107 if (f1 & NETIF_F_HW_CSUM)
4108 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4109 else
4110 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4111 }
4112
4113 return f1 & f2;
4114}
4115
4116static inline netdev_features_t netdev_get_wanted_features(
4117 struct net_device *dev)
4118{
4119 return (dev->features & ~dev->hw_features) | dev->wanted_features;
4120}
4121netdev_features_t netdev_increment_features(netdev_features_t all,
4122 netdev_features_t one, netdev_features_t mask);
4123
4124
4125
4126
4127
4128static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4129 netdev_features_t mask)
4130{
4131 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4132}
4133
4134int __netdev_update_features(struct net_device *dev);
4135void netdev_update_features(struct net_device *dev);
4136void netdev_change_features(struct net_device *dev);
4137
4138void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4139 struct net_device *dev);
4140
4141netdev_features_t passthru_features_check(struct sk_buff *skb,
4142 struct net_device *dev,
4143 netdev_features_t features);
4144netdev_features_t netif_skb_features(struct sk_buff *skb);
4145
4146static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4147{
4148 netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
4149
4150
4151 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
4152 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
4153 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
4154 BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
4155 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
4156 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
4157 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
4158 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
4159 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
4160 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
4161 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
4162 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
4163 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
4164 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
4165 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
4166 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
4167 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
4168
4169 return (features & feature) == feature;
4170}
4171
4172static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
4173{
4174 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
4175 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
4176}
4177
4178static inline bool netif_needs_gso(struct sk_buff *skb,
4179 netdev_features_t features)
4180{
4181 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
4182 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
4183 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
4184}
4185
4186static inline void netif_set_gso_max_size(struct net_device *dev,
4187 unsigned int size)
4188{
4189 dev->gso_max_size = size;
4190}
4191
4192static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
4193 int pulled_hlen, u16 mac_offset,
4194 int mac_len)
4195{
4196 skb->protocol = protocol;
4197 skb->encapsulation = 1;
4198 skb_push(skb, pulled_hlen);
4199 skb_reset_transport_header(skb);
4200 skb->mac_header = mac_offset;
4201 skb->network_header = skb->mac_header + mac_len;
4202 skb->mac_len = mac_len;
4203}
4204
4205static inline bool netif_is_macsec(const struct net_device *dev)
4206{
4207 return dev->priv_flags & IFF_MACSEC;
4208}
4209
4210static inline bool netif_is_macvlan(const struct net_device *dev)
4211{
4212 return dev->priv_flags & IFF_MACVLAN;
4213}
4214
4215static inline bool netif_is_macvlan_port(const struct net_device *dev)
4216{
4217 return dev->priv_flags & IFF_MACVLAN_PORT;
4218}
4219
4220static inline bool netif_is_ipvlan(const struct net_device *dev)
4221{
4222 return dev->priv_flags & IFF_IPVLAN_SLAVE;
4223}
4224
4225static inline bool netif_is_ipvlan_port(const struct net_device *dev)
4226{
4227 return dev->priv_flags & IFF_IPVLAN_MASTER;
4228}
4229
4230static inline bool netif_is_bond_master(const struct net_device *dev)
4231{
4232 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
4233}
4234
4235static inline bool netif_is_bond_slave(const struct net_device *dev)
4236{
4237 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
4238}
4239
4240static inline bool netif_supports_nofcs(struct net_device *dev)
4241{
4242 return dev->priv_flags & IFF_SUPP_NOFCS;
4243}
4244
4245static inline bool netif_is_l3_master(const struct net_device *dev)
4246{
4247 return dev->priv_flags & IFF_L3MDEV_MASTER;
4248}
4249
4250static inline bool netif_is_l3_slave(const struct net_device *dev)
4251{
4252 return dev->priv_flags & IFF_L3MDEV_SLAVE;
4253}
4254
4255static inline bool netif_is_bridge_master(const struct net_device *dev)
4256{
4257 return dev->priv_flags & IFF_EBRIDGE;
4258}
4259
4260static inline bool netif_is_bridge_port(const struct net_device *dev)
4261{
4262 return dev->priv_flags & IFF_BRIDGE_PORT;
4263}
4264
4265static inline bool netif_is_ovs_master(const struct net_device *dev)
4266{
4267 return dev->priv_flags & IFF_OPENVSWITCH;
4268}
4269
4270static inline bool netif_is_ovs_port(const struct net_device *dev)
4271{
4272 return dev->priv_flags & IFF_OVS_DATAPATH;
4273}
4274
4275static inline bool netif_is_team_master(const struct net_device *dev)
4276{
4277 return dev->priv_flags & IFF_TEAM;
4278}
4279
4280static inline bool netif_is_team_port(const struct net_device *dev)
4281{
4282 return dev->priv_flags & IFF_TEAM_PORT;
4283}
4284
4285static inline bool netif_is_lag_master(const struct net_device *dev)
4286{
4287 return netif_is_bond_master(dev) || netif_is_team_master(dev);
4288}
4289
4290static inline bool netif_is_lag_port(const struct net_device *dev)
4291{
4292 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
4293}
4294
4295static inline bool netif_is_rxfh_configured(const struct net_device *dev)
4296{
4297 return dev->priv_flags & IFF_RXFH_CONFIGURED;
4298}
4299
4300
4301static inline void netif_keep_dst(struct net_device *dev)
4302{
4303 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
4304}
4305
4306
4307static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
4308{
4309
4310 return dev->priv_flags & IFF_MACSEC;
4311}
4312
4313extern struct pernet_operations __net_initdata loopback_net_ops;
4314
4315
4316
4317
4318
4319static inline const char *netdev_name(const struct net_device *dev)
4320{
4321 if (!dev->name[0] || strchr(dev->name, '%'))
4322 return "(unnamed net_device)";
4323 return dev->name;
4324}
4325
4326static inline bool netdev_unregistering(const struct net_device *dev)
4327{
4328 return dev->reg_state == NETREG_UNREGISTERING;
4329}
4330
4331static inline const char *netdev_reg_state(const struct net_device *dev)
4332{
4333 switch (dev->reg_state) {
4334 case NETREG_UNINITIALIZED: return " (uninitialized)";
4335 case NETREG_REGISTERED: return "";
4336 case NETREG_UNREGISTERING: return " (unregistering)";
4337 case NETREG_UNREGISTERED: return " (unregistered)";
4338 case NETREG_RELEASED: return " (released)";
4339 case NETREG_DUMMY: return " (dummy)";
4340 }
4341
4342 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
4343 return " (unknown)";
4344}
4345
4346__printf(3, 4)
4347void netdev_printk(const char *level, const struct net_device *dev,
4348 const char *format, ...);
4349__printf(2, 3)
4350void netdev_emerg(const struct net_device *dev, const char *format, ...);
4351__printf(2, 3)
4352void netdev_alert(const struct net_device *dev, const char *format, ...);
4353__printf(2, 3)
4354void netdev_crit(const struct net_device *dev, const char *format, ...);
4355__printf(2, 3)
4356void netdev_err(const struct net_device *dev, const char *format, ...);
4357__printf(2, 3)
4358void netdev_warn(const struct net_device *dev, const char *format, ...);
4359__printf(2, 3)
4360void netdev_notice(const struct net_device *dev, const char *format, ...);
4361__printf(2, 3)
4362void netdev_info(const struct net_device *dev, const char *format, ...);
4363
4364#define netdev_level_once(level, dev, fmt, ...) \
4365do { \
4366 static bool __print_once __read_mostly; \
4367 \
4368 if (!__print_once) { \
4369 __print_once = true; \
4370 netdev_printk(level, dev, fmt, ##__VA_ARGS__); \
4371 } \
4372} while (0)
4373
4374#define netdev_emerg_once(dev, fmt, ...) \
4375 netdev_level_once(KERN_EMERG, dev, fmt, ##__VA_ARGS__)
4376#define netdev_alert_once(dev, fmt, ...) \
4377 netdev_level_once(KERN_ALERT, dev, fmt, ##__VA_ARGS__)
4378#define netdev_crit_once(dev, fmt, ...) \
4379 netdev_level_once(KERN_CRIT, dev, fmt, ##__VA_ARGS__)
4380#define netdev_err_once(dev, fmt, ...) \
4381 netdev_level_once(KERN_ERR, dev, fmt, ##__VA_ARGS__)
4382#define netdev_warn_once(dev, fmt, ...) \
4383 netdev_level_once(KERN_WARNING, dev, fmt, ##__VA_ARGS__)
4384#define netdev_notice_once(dev, fmt, ...) \
4385 netdev_level_once(KERN_NOTICE, dev, fmt, ##__VA_ARGS__)
4386#define netdev_info_once(dev, fmt, ...) \
4387 netdev_level_once(KERN_INFO, dev, fmt, ##__VA_ARGS__)
4388
4389#define MODULE_ALIAS_NETDEV(device) \
4390 MODULE_ALIAS("netdev-" device)
4391
4392#if defined(CONFIG_DYNAMIC_DEBUG)
4393#define netdev_dbg(__dev, format, args...) \
4394do { \
4395 dynamic_netdev_dbg(__dev, format, ##args); \
4396} while (0)
4397#elif defined(DEBUG)
4398#define netdev_dbg(__dev, format, args...) \
4399 netdev_printk(KERN_DEBUG, __dev, format, ##args)
4400#else
4401#define netdev_dbg(__dev, format, args...) \
4402({ \
4403 if (0) \
4404 netdev_printk(KERN_DEBUG, __dev, format, ##args); \
4405})
4406#endif
4407
4408#if defined(VERBOSE_DEBUG)
4409#define netdev_vdbg netdev_dbg
4410#else
4411
4412#define netdev_vdbg(dev, format, args...) \
4413({ \
4414 if (0) \
4415 netdev_printk(KERN_DEBUG, dev, format, ##args); \
4416 0; \
4417})
4418#endif
4419
4420
4421
4422
4423
4424
4425#define netdev_WARN(dev, format, args...) \
4426 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
4427 netdev_reg_state(dev), ##args)
4428
4429#define netdev_WARN_ONCE(dev, format, args...) \
4430 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
4431 netdev_reg_state(dev), ##args)
4432
4433
4434
4435#define netif_printk(priv, type, level, dev, fmt, args...) \
4436do { \
4437 if (netif_msg_##type(priv)) \
4438 netdev_printk(level, (dev), fmt, ##args); \
4439} while (0)
4440
4441#define netif_level(level, priv, type, dev, fmt, args...) \
4442do { \
4443 if (netif_msg_##type(priv)) \
4444 netdev_##level(dev, fmt, ##args); \
4445} while (0)
4446
4447#define netif_emerg(priv, type, dev, fmt, args...) \
4448 netif_level(emerg, priv, type, dev, fmt, ##args)
4449#define netif_alert(priv, type, dev, fmt, args...) \
4450 netif_level(alert, priv, type, dev, fmt, ##args)
4451#define netif_crit(priv, type, dev, fmt, args...) \
4452 netif_level(crit, priv, type, dev, fmt, ##args)
4453#define netif_err(priv, type, dev, fmt, args...) \
4454 netif_level(err, priv, type, dev, fmt, ##args)
4455#define netif_warn(priv, type, dev, fmt, args...) \
4456 netif_level(warn, priv, type, dev, fmt, ##args)
4457#define netif_notice(priv, type, dev, fmt, args...) \
4458 netif_level(notice, priv, type, dev, fmt, ##args)
4459#define netif_info(priv, type, dev, fmt, args...) \
4460 netif_level(info, priv, type, dev, fmt, ##args)
4461
4462#if defined(CONFIG_DYNAMIC_DEBUG)
4463#define netif_dbg(priv, type, netdev, format, args...) \
4464do { \
4465 if (netif_msg_##type(priv)) \
4466 dynamic_netdev_dbg(netdev, format, ##args); \
4467} while (0)
4468#elif defined(DEBUG)
4469#define netif_dbg(priv, type, dev, format, args...) \
4470 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
4471#else
4472#define netif_dbg(priv, type, dev, format, args...) \
4473({ \
4474 if (0) \
4475 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4476 0; \
4477})
4478#endif
4479
4480
4481#define netif_cond_dbg(priv, type, netdev, cond, level, fmt, args...) \
4482 do { \
4483 if (cond) \
4484 netif_dbg(priv, type, netdev, fmt, ##args); \
4485 else \
4486 netif_ ## level(priv, type, netdev, fmt, ##args); \
4487 } while (0)
4488
4489#if defined(VERBOSE_DEBUG)
4490#define netif_vdbg netif_dbg
4491#else
4492#define netif_vdbg(priv, type, dev, format, args...) \
4493({ \
4494 if (0) \
4495 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4496 0; \
4497})
4498#endif
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519#define PTYPE_HASH_SIZE (16)
4520#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
4521
4522#endif
4523