1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef _LINUX_NETDEVICE_H
26#define _LINUX_NETDEVICE_H
27
28#include <linux/timer.h>
29#include <linux/bug.h>
30#include <linux/delay.h>
31#include <linux/atomic.h>
32#include <linux/prefetch.h>
33#include <asm/cache.h>
34#include <asm/byteorder.h>
35
36#include <linux/percpu.h>
37#include <linux/rculist.h>
38#include <linux/dmaengine.h>
39#include <linux/workqueue.h>
40#include <linux/dynamic_queue_limits.h>
41
42#include <linux/ethtool.h>
43#include <net/net_namespace.h>
44#include <net/dsa.h>
45#ifdef CONFIG_DCB
46#include <net/dcbnl.h>
47#endif
48#include <net/netprio_cgroup.h>
49
50#include <linux/netdev_features.h>
51#include <linux/neighbour.h>
52#include <uapi/linux/netdevice.h>
53#include <uapi/linux/if_bonding.h>
54
55struct netpoll_info;
56struct device;
57struct phy_device;
58
59struct wireless_dev;
60
61struct wpan_dev;
62struct mpls_dev;
63
64void netdev_set_default_ethtool_ops(struct net_device *dev,
65 const struct ethtool_ops *ops);
66
67
68#define NET_RX_SUCCESS 0
69#define NET_RX_DROP 1
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89#define NET_XMIT_SUCCESS 0x00
90#define NET_XMIT_DROP 0x01
91#define NET_XMIT_CN 0x02
92#define NET_XMIT_POLICED 0x03
93#define NET_XMIT_MASK 0x0f
94
95
96
97
98#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
99#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
100
101
102#define NETDEV_TX_MASK 0xf0
103
104enum netdev_tx {
105 __NETDEV_TX_MIN = INT_MIN,
106 NETDEV_TX_OK = 0x00,
107 NETDEV_TX_BUSY = 0x10,
108 NETDEV_TX_LOCKED = 0x20,
109};
110typedef enum netdev_tx netdev_tx_t;
111
112
113
114
115
116static inline bool dev_xmit_complete(int rc)
117{
118
119
120
121
122
123
124 if (likely(rc < NET_XMIT_MASK))
125 return true;
126
127 return false;
128}
129
130
131
132
133
134
135#if defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
136# if defined(CONFIG_MAC80211_MESH)
137# define LL_MAX_HEADER 128
138# else
139# define LL_MAX_HEADER 96
140# endif
141#else
142# define LL_MAX_HEADER 32
143#endif
144
145#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
146 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
147#define MAX_HEADER LL_MAX_HEADER
148#else
149#define MAX_HEADER (LL_MAX_HEADER + 48)
150#endif
151
152
153
154
155
156
157struct net_device_stats {
158 unsigned long rx_packets;
159 unsigned long tx_packets;
160 unsigned long rx_bytes;
161 unsigned long tx_bytes;
162 unsigned long rx_errors;
163 unsigned long tx_errors;
164 unsigned long rx_dropped;
165 unsigned long tx_dropped;
166 unsigned long multicast;
167 unsigned long collisions;
168 unsigned long rx_length_errors;
169 unsigned long rx_over_errors;
170 unsigned long rx_crc_errors;
171 unsigned long rx_frame_errors;
172 unsigned long rx_fifo_errors;
173 unsigned long rx_missed_errors;
174 unsigned long tx_aborted_errors;
175 unsigned long tx_carrier_errors;
176 unsigned long tx_fifo_errors;
177 unsigned long tx_heartbeat_errors;
178 unsigned long tx_window_errors;
179 unsigned long rx_compressed;
180 unsigned long tx_compressed;
181};
182
183
184#include <linux/cache.h>
185#include <linux/skbuff.h>
186
187#ifdef CONFIG_RPS
188#include <linux/static_key.h>
189extern struct static_key rps_needed;
190#endif
191
192struct neighbour;
193struct neigh_parms;
194struct sk_buff;
195
196struct netdev_hw_addr {
197 struct list_head list;
198 unsigned char addr[MAX_ADDR_LEN];
199 unsigned char type;
200#define NETDEV_HW_ADDR_T_LAN 1
201#define NETDEV_HW_ADDR_T_SAN 2
202#define NETDEV_HW_ADDR_T_SLAVE 3
203#define NETDEV_HW_ADDR_T_UNICAST 4
204#define NETDEV_HW_ADDR_T_MULTICAST 5
205 bool global_use;
206 int sync_cnt;
207 int refcount;
208 int synced;
209 struct rcu_head rcu_head;
210};
211
212struct netdev_hw_addr_list {
213 struct list_head list;
214 int count;
215};
216
217#define netdev_hw_addr_list_count(l) ((l)->count)
218#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
219#define netdev_hw_addr_list_for_each(ha, l) \
220 list_for_each_entry(ha, &(l)->list, list)
221
222#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
223#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
224#define netdev_for_each_uc_addr(ha, dev) \
225 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
226
227#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
228#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
229#define netdev_for_each_mc_addr(ha, dev) \
230 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
231
232struct hh_cache {
233 u16 hh_len;
234 u16 __pad;
235 seqlock_t hh_lock;
236
237
238#define HH_DATA_MOD 16
239#define HH_DATA_OFF(__len) \
240 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
241#define HH_DATA_ALIGN(__len) \
242 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
243 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
244};
245
246
247
248
249
250
251
252
253
254#define LL_RESERVED_SPACE(dev) \
255 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
256#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
257 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
258
259struct header_ops {
260 int (*create) (struct sk_buff *skb, struct net_device *dev,
261 unsigned short type, const void *daddr,
262 const void *saddr, unsigned int len);
263 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
264 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
265 void (*cache_update)(struct hh_cache *hh,
266 const struct net_device *dev,
267 const unsigned char *haddr);
268};
269
270
271
272
273
274
275enum netdev_state_t {
276 __LINK_STATE_START,
277 __LINK_STATE_PRESENT,
278 __LINK_STATE_NOCARRIER,
279 __LINK_STATE_LINKWATCH_PENDING,
280 __LINK_STATE_DORMANT,
281};
282
283
284
285
286
287
288struct netdev_boot_setup {
289 char name[IFNAMSIZ];
290 struct ifmap map;
291};
292#define NETDEV_BOOT_SETUP_MAX 8
293
294int __init netdev_boot_setup(char *str);
295
296
297
298
299struct napi_struct {
300
301
302
303
304
305
306 struct list_head poll_list;
307
308 unsigned long state;
309 int weight;
310 unsigned int gro_count;
311 int (*poll)(struct napi_struct *, int);
312#ifdef CONFIG_NETPOLL
313 spinlock_t poll_lock;
314 int poll_owner;
315#endif
316 struct net_device *dev;
317 struct sk_buff *gro_list;
318 struct sk_buff *skb;
319 struct hrtimer timer;
320 struct list_head dev_list;
321 struct hlist_node napi_hash_node;
322 unsigned int napi_id;
323};
324
325enum {
326 NAPI_STATE_SCHED,
327 NAPI_STATE_DISABLE,
328 NAPI_STATE_NPSVC,
329 NAPI_STATE_HASHED,
330};
331
332enum gro_result {
333 GRO_MERGED,
334 GRO_MERGED_FREE,
335 GRO_HELD,
336 GRO_NORMAL,
337 GRO_DROP,
338};
339typedef enum gro_result gro_result_t;
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382enum rx_handler_result {
383 RX_HANDLER_CONSUMED,
384 RX_HANDLER_ANOTHER,
385 RX_HANDLER_EXACT,
386 RX_HANDLER_PASS,
387};
388typedef enum rx_handler_result rx_handler_result_t;
389typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
390
391void __napi_schedule(struct napi_struct *n);
392void __napi_schedule_irqoff(struct napi_struct *n);
393
394static inline bool napi_disable_pending(struct napi_struct *n)
395{
396 return test_bit(NAPI_STATE_DISABLE, &n->state);
397}
398
399
400
401
402
403
404
405
406
407
408static inline bool napi_schedule_prep(struct napi_struct *n)
409{
410 return !napi_disable_pending(n) &&
411 !test_and_set_bit(NAPI_STATE_SCHED, &n->state);
412}
413
414
415
416
417
418
419
420
421static inline void napi_schedule(struct napi_struct *n)
422{
423 if (napi_schedule_prep(n))
424 __napi_schedule(n);
425}
426
427
428
429
430
431
432
433static inline void napi_schedule_irqoff(struct napi_struct *n)
434{
435 if (napi_schedule_prep(n))
436 __napi_schedule_irqoff(n);
437}
438
439
440static inline bool napi_reschedule(struct napi_struct *napi)
441{
442 if (napi_schedule_prep(napi)) {
443 __napi_schedule(napi);
444 return true;
445 }
446 return false;
447}
448
449void __napi_complete(struct napi_struct *n);
450void napi_complete_done(struct napi_struct *n, int work_done);
451
452
453
454
455
456
457
458static inline void napi_complete(struct napi_struct *n)
459{
460 return napi_complete_done(n, 0);
461}
462
463
464
465
466
467
468
469
470struct napi_struct *napi_by_id(unsigned int napi_id);
471
472
473
474
475
476
477
478void napi_hash_add(struct napi_struct *napi);
479
480
481
482
483
484
485
486
487void napi_hash_del(struct napi_struct *napi);
488
489
490
491
492
493
494
495
496void napi_disable(struct napi_struct *n);
497
498
499
500
501
502
503
504
505static inline void napi_enable(struct napi_struct *n)
506{
507 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
508 smp_mb__before_atomic();
509 clear_bit(NAPI_STATE_SCHED, &n->state);
510 clear_bit(NAPI_STATE_NPSVC, &n->state);
511}
512
513#ifdef CONFIG_SMP
514
515
516
517
518
519
520
521
522static inline void napi_synchronize(const struct napi_struct *n)
523{
524 while (test_bit(NAPI_STATE_SCHED, &n->state))
525 msleep(1);
526}
527#else
528# define napi_synchronize(n) barrier()
529#endif
530
531enum netdev_queue_state_t {
532 __QUEUE_STATE_DRV_XOFF,
533 __QUEUE_STATE_STACK_XOFF,
534 __QUEUE_STATE_FROZEN,
535};
536
537#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
538#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
539#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
540
541#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
542#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
543 QUEUE_STATE_FROZEN)
544#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
545 QUEUE_STATE_FROZEN)
546
547
548
549
550
551
552
553
554
555
556
557struct netdev_queue {
558
559
560
561 struct net_device *dev;
562 struct Qdisc __rcu *qdisc;
563 struct Qdisc *qdisc_sleeping;
564#ifdef CONFIG_SYSFS
565 struct kobject kobj;
566#endif
567#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
568 int numa_node;
569#endif
570
571
572
573 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
574 int xmit_lock_owner;
575
576
577
578 unsigned long trans_start;
579
580
581
582
583
584 unsigned long trans_timeout;
585
586 unsigned long state;
587
588#ifdef CONFIG_BQL
589 struct dql dql;
590#endif
591 unsigned long tx_maxrate;
592} ____cacheline_aligned_in_smp;
593
594static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
595{
596#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
597 return q->numa_node;
598#else
599 return NUMA_NO_NODE;
600#endif
601}
602
603static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
604{
605#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
606 q->numa_node = node;
607#endif
608}
609
610#ifdef CONFIG_RPS
611
612
613
614
615struct rps_map {
616 unsigned int len;
617 struct rcu_head rcu;
618 u16 cpus[0];
619};
620#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
621
622
623
624
625
626
627struct rps_dev_flow {
628 u16 cpu;
629 u16 filter;
630 unsigned int last_qtail;
631};
632#define RPS_NO_FILTER 0xffff
633
634
635
636
637struct rps_dev_flow_table {
638 unsigned int mask;
639 struct rcu_head rcu;
640 struct rps_dev_flow flows[0];
641};
642#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
643 ((_num) * sizeof(struct rps_dev_flow)))
644
645
646
647
648
649
650
651
652
653
654
655struct rps_sock_flow_table {
656 u32 mask;
657
658 u32 ents[0] ____cacheline_aligned_in_smp;
659};
660#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
661
662#define RPS_NO_CPU 0xffff
663
664extern u32 rps_cpu_mask;
665extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
666
667static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
668 u32 hash)
669{
670 if (table && hash) {
671 unsigned int index = hash & table->mask;
672 u32 val = hash & ~rps_cpu_mask;
673
674
675 val |= raw_smp_processor_id();
676
677 if (table->ents[index] != val)
678 table->ents[index] = val;
679 }
680}
681
682#ifdef CONFIG_RFS_ACCEL
683bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
684 u16 filter_id);
685#endif
686#endif
687
688
689struct netdev_rx_queue {
690#ifdef CONFIG_RPS
691 struct rps_map __rcu *rps_map;
692 struct rps_dev_flow_table __rcu *rps_flow_table;
693#endif
694 struct kobject kobj;
695 struct net_device *dev;
696} ____cacheline_aligned_in_smp;
697
698
699
700
701struct rx_queue_attribute {
702 struct attribute attr;
703 ssize_t (*show)(struct netdev_rx_queue *queue,
704 struct rx_queue_attribute *attr, char *buf);
705 ssize_t (*store)(struct netdev_rx_queue *queue,
706 struct rx_queue_attribute *attr, const char *buf, size_t len);
707};
708
709#ifdef CONFIG_XPS
710
711
712
713
714struct xps_map {
715 unsigned int len;
716 unsigned int alloc_len;
717 struct rcu_head rcu;
718 u16 queues[0];
719};
720#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
721#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
722 - sizeof(struct xps_map)) / sizeof(u16))
723
724
725
726
727struct xps_dev_maps {
728 struct rcu_head rcu;
729 struct xps_map __rcu *cpu_map[0];
730};
731#define XPS_DEV_MAPS_SIZE (sizeof(struct xps_dev_maps) + \
732 (nr_cpu_ids * sizeof(struct xps_map *)))
733#endif
734
735#define TC_MAX_QUEUE 16
736#define TC_BITMASK 15
737
738struct netdev_tc_txq {
739 u16 count;
740 u16 offset;
741};
742
743#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
744
745
746
747
748struct netdev_fcoe_hbainfo {
749 char manufacturer[64];
750 char serial_number[64];
751 char hardware_version[64];
752 char driver_version[64];
753 char optionrom_version[64];
754 char firmware_version[64];
755 char model[256];
756 char model_description[256];
757};
758#endif
759
760#define MAX_PHYS_ITEM_ID_LEN 32
761
762
763
764
765struct netdev_phys_item_id {
766 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
767 unsigned char id_len;
768};
769
770static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
771 struct netdev_phys_item_id *b)
772{
773 return a->id_len == b->id_len &&
774 memcmp(a->id, b->id, a->id_len) == 0;
775}
776
777typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
778 struct sk_buff *skb);
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064struct net_device_ops {
1065 int (*ndo_init)(struct net_device *dev);
1066 void (*ndo_uninit)(struct net_device *dev);
1067 int (*ndo_open)(struct net_device *dev);
1068 int (*ndo_stop)(struct net_device *dev);
1069 netdev_tx_t (*ndo_start_xmit) (struct sk_buff *skb,
1070 struct net_device *dev);
1071 u16 (*ndo_select_queue)(struct net_device *dev,
1072 struct sk_buff *skb,
1073 void *accel_priv,
1074 select_queue_fallback_t fallback);
1075 void (*ndo_change_rx_flags)(struct net_device *dev,
1076 int flags);
1077 void (*ndo_set_rx_mode)(struct net_device *dev);
1078 int (*ndo_set_mac_address)(struct net_device *dev,
1079 void *addr);
1080 int (*ndo_validate_addr)(struct net_device *dev);
1081 int (*ndo_do_ioctl)(struct net_device *dev,
1082 struct ifreq *ifr, int cmd);
1083 int (*ndo_set_config)(struct net_device *dev,
1084 struct ifmap *map);
1085 int (*ndo_change_mtu)(struct net_device *dev,
1086 int new_mtu);
1087 int (*ndo_neigh_setup)(struct net_device *dev,
1088 struct neigh_parms *);
1089 void (*ndo_tx_timeout) (struct net_device *dev);
1090
1091 struct rtnl_link_stats64* (*ndo_get_stats64)(struct net_device *dev,
1092 struct rtnl_link_stats64 *storage);
1093 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1094
1095 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1096 __be16 proto, u16 vid);
1097 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1098 __be16 proto, u16 vid);
1099#ifdef CONFIG_NET_POLL_CONTROLLER
1100 void (*ndo_poll_controller)(struct net_device *dev);
1101 int (*ndo_netpoll_setup)(struct net_device *dev,
1102 struct netpoll_info *info);
1103 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1104#endif
1105#ifdef CONFIG_NET_RX_BUSY_POLL
1106 int (*ndo_busy_poll)(struct napi_struct *dev);
1107#endif
1108 int (*ndo_set_vf_mac)(struct net_device *dev,
1109 int queue, u8 *mac);
1110 int (*ndo_set_vf_vlan)(struct net_device *dev,
1111 int queue, u16 vlan, u8 qos);
1112 int (*ndo_set_vf_rate)(struct net_device *dev,
1113 int vf, int min_tx_rate,
1114 int max_tx_rate);
1115 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1116 int vf, bool setting);
1117 int (*ndo_set_vf_trust)(struct net_device *dev,
1118 int vf, bool setting);
1119 int (*ndo_get_vf_config)(struct net_device *dev,
1120 int vf,
1121 struct ifla_vf_info *ivf);
1122 int (*ndo_set_vf_link_state)(struct net_device *dev,
1123 int vf, int link_state);
1124 int (*ndo_get_vf_stats)(struct net_device *dev,
1125 int vf,
1126 struct ifla_vf_stats
1127 *vf_stats);
1128 int (*ndo_set_vf_port)(struct net_device *dev,
1129 int vf,
1130 struct nlattr *port[]);
1131 int (*ndo_get_vf_port)(struct net_device *dev,
1132 int vf, struct sk_buff *skb);
1133 int (*ndo_set_vf_rss_query_en)(
1134 struct net_device *dev,
1135 int vf, bool setting);
1136 int (*ndo_setup_tc)(struct net_device *dev, u8 tc);
1137#if IS_ENABLED(CONFIG_FCOE)
1138 int (*ndo_fcoe_enable)(struct net_device *dev);
1139 int (*ndo_fcoe_disable)(struct net_device *dev);
1140 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1141 u16 xid,
1142 struct scatterlist *sgl,
1143 unsigned int sgc);
1144 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1145 u16 xid);
1146 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1147 u16 xid,
1148 struct scatterlist *sgl,
1149 unsigned int sgc);
1150 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1151 struct netdev_fcoe_hbainfo *hbainfo);
1152#endif
1153
1154#if IS_ENABLED(CONFIG_LIBFCOE)
1155#define NETDEV_FCOE_WWNN 0
1156#define NETDEV_FCOE_WWPN 1
1157 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1158 u64 *wwn, int type);
1159#endif
1160
1161#ifdef CONFIG_RFS_ACCEL
1162 int (*ndo_rx_flow_steer)(struct net_device *dev,
1163 const struct sk_buff *skb,
1164 u16 rxq_index,
1165 u32 flow_id);
1166#endif
1167 int (*ndo_add_slave)(struct net_device *dev,
1168 struct net_device *slave_dev);
1169 int (*ndo_del_slave)(struct net_device *dev,
1170 struct net_device *slave_dev);
1171 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1172 netdev_features_t features);
1173 int (*ndo_set_features)(struct net_device *dev,
1174 netdev_features_t features);
1175 int (*ndo_neigh_construct)(struct neighbour *n);
1176 void (*ndo_neigh_destroy)(struct neighbour *n);
1177
1178 int (*ndo_fdb_add)(struct ndmsg *ndm,
1179 struct nlattr *tb[],
1180 struct net_device *dev,
1181 const unsigned char *addr,
1182 u16 vid,
1183 u16 flags);
1184 int (*ndo_fdb_del)(struct ndmsg *ndm,
1185 struct nlattr *tb[],
1186 struct net_device *dev,
1187 const unsigned char *addr,
1188 u16 vid);
1189 int (*ndo_fdb_dump)(struct sk_buff *skb,
1190 struct netlink_callback *cb,
1191 struct net_device *dev,
1192 struct net_device *filter_dev,
1193 int idx);
1194
1195 int (*ndo_bridge_setlink)(struct net_device *dev,
1196 struct nlmsghdr *nlh,
1197 u16 flags);
1198 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1199 u32 pid, u32 seq,
1200 struct net_device *dev,
1201 u32 filter_mask,
1202 int nlflags);
1203 int (*ndo_bridge_dellink)(struct net_device *dev,
1204 struct nlmsghdr *nlh,
1205 u16 flags);
1206 int (*ndo_change_carrier)(struct net_device *dev,
1207 bool new_carrier);
1208 int (*ndo_get_phys_port_id)(struct net_device *dev,
1209 struct netdev_phys_item_id *ppid);
1210 int (*ndo_get_phys_port_name)(struct net_device *dev,
1211 char *name, size_t len);
1212 void (*ndo_add_vxlan_port)(struct net_device *dev,
1213 sa_family_t sa_family,
1214 __be16 port);
1215 void (*ndo_del_vxlan_port)(struct net_device *dev,
1216 sa_family_t sa_family,
1217 __be16 port);
1218
1219 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1220 struct net_device *dev);
1221 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1222 void *priv);
1223
1224 netdev_tx_t (*ndo_dfwd_start_xmit) (struct sk_buff *skb,
1225 struct net_device *dev,
1226 void *priv);
1227 int (*ndo_get_lock_subclass)(struct net_device *dev);
1228 netdev_features_t (*ndo_features_check) (struct sk_buff *skb,
1229 struct net_device *dev,
1230 netdev_features_t features);
1231 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1232 int queue_index,
1233 u32 maxrate);
1234 int (*ndo_get_iflink)(const struct net_device *dev);
1235 int (*ndo_change_proto_down)(struct net_device *dev,
1236 bool proto_down);
1237 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1238 struct sk_buff *skb);
1239};
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275enum netdev_priv_flags {
1276 IFF_802_1Q_VLAN = 1<<0,
1277 IFF_EBRIDGE = 1<<1,
1278 IFF_BONDING = 1<<2,
1279 IFF_ISATAP = 1<<3,
1280 IFF_WAN_HDLC = 1<<4,
1281 IFF_XMIT_DST_RELEASE = 1<<5,
1282 IFF_DONT_BRIDGE = 1<<6,
1283 IFF_DISABLE_NETPOLL = 1<<7,
1284 IFF_MACVLAN_PORT = 1<<8,
1285 IFF_BRIDGE_PORT = 1<<9,
1286 IFF_OVS_DATAPATH = 1<<10,
1287 IFF_TX_SKB_SHARING = 1<<11,
1288 IFF_UNICAST_FLT = 1<<12,
1289 IFF_TEAM_PORT = 1<<13,
1290 IFF_SUPP_NOFCS = 1<<14,
1291 IFF_LIVE_ADDR_CHANGE = 1<<15,
1292 IFF_MACVLAN = 1<<16,
1293 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1294 IFF_IPVLAN_MASTER = 1<<18,
1295 IFF_IPVLAN_SLAVE = 1<<19,
1296 IFF_L3MDEV_MASTER = 1<<20,
1297 IFF_NO_QUEUE = 1<<21,
1298 IFF_OPENVSWITCH = 1<<22,
1299 IFF_L3MDEV_SLAVE = 1<<23,
1300};
1301
1302#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
1303#define IFF_EBRIDGE IFF_EBRIDGE
1304#define IFF_BONDING IFF_BONDING
1305#define IFF_ISATAP IFF_ISATAP
1306#define IFF_WAN_HDLC IFF_WAN_HDLC
1307#define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
1308#define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
1309#define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
1310#define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
1311#define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
1312#define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
1313#define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
1314#define IFF_UNICAST_FLT IFF_UNICAST_FLT
1315#define IFF_TEAM_PORT IFF_TEAM_PORT
1316#define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
1317#define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
1318#define IFF_MACVLAN IFF_MACVLAN
1319#define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
1320#define IFF_IPVLAN_MASTER IFF_IPVLAN_MASTER
1321#define IFF_IPVLAN_SLAVE IFF_IPVLAN_SLAVE
1322#define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
1323#define IFF_NO_QUEUE IFF_NO_QUEUE
1324#define IFF_OPENVSWITCH IFF_OPENVSWITCH
1325#define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538struct net_device {
1539 char name[IFNAMSIZ];
1540 struct hlist_node name_hlist;
1541 char *ifalias;
1542
1543
1544
1545
1546 unsigned long mem_end;
1547 unsigned long mem_start;
1548 unsigned long base_addr;
1549 int irq;
1550
1551 atomic_t carrier_changes;
1552
1553
1554
1555
1556
1557
1558
1559 unsigned long state;
1560
1561 struct list_head dev_list;
1562 struct list_head napi_list;
1563 struct list_head unreg_list;
1564 struct list_head close_list;
1565 struct list_head ptype_all;
1566 struct list_head ptype_specific;
1567
1568 struct {
1569 struct list_head upper;
1570 struct list_head lower;
1571 } adj_list;
1572
1573 struct {
1574 struct list_head upper;
1575 struct list_head lower;
1576 } all_adj_list;
1577
1578 netdev_features_t features;
1579 netdev_features_t hw_features;
1580 netdev_features_t wanted_features;
1581 netdev_features_t vlan_features;
1582 netdev_features_t hw_enc_features;
1583 netdev_features_t mpls_features;
1584
1585 int ifindex;
1586 int group;
1587
1588 struct net_device_stats stats;
1589
1590 atomic_long_t rx_dropped;
1591 atomic_long_t tx_dropped;
1592
1593#ifdef CONFIG_WIRELESS_EXT
1594 const struct iw_handler_def * wireless_handlers;
1595 struct iw_public_data * wireless_data;
1596#endif
1597 const struct net_device_ops *netdev_ops;
1598 const struct ethtool_ops *ethtool_ops;
1599#ifdef CONFIG_NET_SWITCHDEV
1600 const struct switchdev_ops *switchdev_ops;
1601#endif
1602#ifdef CONFIG_NET_L3_MASTER_DEV
1603 const struct l3mdev_ops *l3mdev_ops;
1604#endif
1605
1606 const struct header_ops *header_ops;
1607
1608 unsigned int flags;
1609 unsigned int priv_flags;
1610
1611 unsigned short gflags;
1612 unsigned short padded;
1613
1614 unsigned char operstate;
1615 unsigned char link_mode;
1616
1617 unsigned char if_port;
1618 unsigned char dma;
1619
1620 unsigned int mtu;
1621 unsigned short type;
1622 unsigned short hard_header_len;
1623
1624 unsigned short needed_headroom;
1625 unsigned short needed_tailroom;
1626
1627
1628 unsigned char perm_addr[MAX_ADDR_LEN];
1629 unsigned char addr_assign_type;
1630 unsigned char addr_len;
1631 unsigned short neigh_priv_len;
1632 unsigned short dev_id;
1633 unsigned short dev_port;
1634 spinlock_t addr_list_lock;
1635 unsigned char name_assign_type;
1636 bool uc_promisc;
1637 struct netdev_hw_addr_list uc;
1638 struct netdev_hw_addr_list mc;
1639 struct netdev_hw_addr_list dev_addrs;
1640
1641#ifdef CONFIG_SYSFS
1642 struct kset *queues_kset;
1643#endif
1644 unsigned int promiscuity;
1645 unsigned int allmulti;
1646
1647
1648
1649
1650#if IS_ENABLED(CONFIG_VLAN_8021Q)
1651 struct vlan_info __rcu *vlan_info;
1652#endif
1653#if IS_ENABLED(CONFIG_NET_DSA)
1654 struct dsa_switch_tree *dsa_ptr;
1655#endif
1656#if IS_ENABLED(CONFIG_TIPC)
1657 struct tipc_bearer __rcu *tipc_ptr;
1658#endif
1659 void *atalk_ptr;
1660 struct in_device __rcu *ip_ptr;
1661 struct dn_dev __rcu *dn_ptr;
1662 struct inet6_dev __rcu *ip6_ptr;
1663 void *ax25_ptr;
1664 struct wireless_dev *ieee80211_ptr;
1665 struct wpan_dev *ieee802154_ptr;
1666#if IS_ENABLED(CONFIG_MPLS_ROUTING)
1667 struct mpls_dev __rcu *mpls_ptr;
1668#endif
1669
1670
1671
1672
1673 unsigned long last_rx;
1674
1675
1676 unsigned char *dev_addr;
1677
1678
1679#ifdef CONFIG_SYSFS
1680 struct netdev_rx_queue *_rx;
1681
1682 unsigned int num_rx_queues;
1683 unsigned int real_num_rx_queues;
1684
1685#endif
1686
1687 unsigned long gro_flush_timeout;
1688 rx_handler_func_t __rcu *rx_handler;
1689 void __rcu *rx_handler_data;
1690
1691#ifdef CONFIG_NET_CLS_ACT
1692 struct tcf_proto __rcu *ingress_cl_list;
1693#endif
1694 struct netdev_queue __rcu *ingress_queue;
1695#ifdef CONFIG_NETFILTER_INGRESS
1696 struct list_head nf_hooks_ingress;
1697#endif
1698
1699 unsigned char broadcast[MAX_ADDR_LEN];
1700#ifdef CONFIG_RFS_ACCEL
1701 struct cpu_rmap *rx_cpu_rmap;
1702#endif
1703 struct hlist_node index_hlist;
1704
1705
1706
1707
1708 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
1709 unsigned int num_tx_queues;
1710 unsigned int real_num_tx_queues;
1711 struct Qdisc *qdisc;
1712 unsigned long tx_queue_len;
1713 spinlock_t tx_global_lock;
1714 int watchdog_timeo;
1715
1716#ifdef CONFIG_XPS
1717 struct xps_dev_maps __rcu *xps_maps;
1718#endif
1719
1720#ifdef CONFIG_NET_SWITCHDEV
1721 u32 offload_fwd_mark;
1722#endif
1723
1724
1725
1726
1727
1728
1729
1730 unsigned long trans_start;
1731
1732 struct timer_list watchdog_timer;
1733
1734 int __percpu *pcpu_refcnt;
1735 struct list_head todo_list;
1736
1737 struct list_head link_watch_list;
1738
1739 enum { NETREG_UNINITIALIZED=0,
1740 NETREG_REGISTERED,
1741 NETREG_UNREGISTERING,
1742 NETREG_UNREGISTERED,
1743 NETREG_RELEASED,
1744 NETREG_DUMMY,
1745 } reg_state:8;
1746
1747 bool dismantle;
1748
1749 enum {
1750 RTNL_LINK_INITIALIZED,
1751 RTNL_LINK_INITIALIZING,
1752 } rtnl_link_state:16;
1753
1754 void (*destructor)(struct net_device *dev);
1755
1756#ifdef CONFIG_NETPOLL
1757 struct netpoll_info __rcu *npinfo;
1758#endif
1759
1760 possible_net_t nd_net;
1761
1762
1763 union {
1764 void *ml_priv;
1765 struct pcpu_lstats __percpu *lstats;
1766 struct pcpu_sw_netstats __percpu *tstats;
1767 struct pcpu_dstats __percpu *dstats;
1768 struct pcpu_vstats __percpu *vstats;
1769 };
1770
1771 struct garp_port __rcu *garp_port;
1772 struct mrp_port __rcu *mrp_port;
1773
1774 struct device dev;
1775 const struct attribute_group *sysfs_groups[4];
1776 const struct attribute_group *sysfs_rx_queue_group;
1777
1778 const struct rtnl_link_ops *rtnl_link_ops;
1779
1780
1781#define GSO_MAX_SIZE 65536
1782 unsigned int gso_max_size;
1783#define GSO_MAX_SEGS 65535
1784 u16 gso_max_segs;
1785 u16 gso_min_segs;
1786#ifdef CONFIG_DCB
1787 const struct dcbnl_rtnl_ops *dcbnl_ops;
1788#endif
1789 u8 num_tc;
1790 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1791 u8 prio_tc_map[TC_BITMASK + 1];
1792
1793#if IS_ENABLED(CONFIG_FCOE)
1794 unsigned int fcoe_ddp_xid;
1795#endif
1796#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
1797 struct netprio_map __rcu *priomap;
1798#endif
1799 struct phy_device *phydev;
1800 struct lock_class_key *qdisc_tx_busylock;
1801 bool proto_down;
1802};
1803#define to_net_dev(d) container_of(d, struct net_device, dev)
1804
1805#define NETDEV_ALIGN 32
1806
1807static inline
1808int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1809{
1810 return dev->prio_tc_map[prio & TC_BITMASK];
1811}
1812
1813static inline
1814int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1815{
1816 if (tc >= dev->num_tc)
1817 return -EINVAL;
1818
1819 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
1820 return 0;
1821}
1822
1823static inline
1824void netdev_reset_tc(struct net_device *dev)
1825{
1826 dev->num_tc = 0;
1827 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
1828 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
1829}
1830
1831static inline
1832int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
1833{
1834 if (tc >= dev->num_tc)
1835 return -EINVAL;
1836
1837 dev->tc_to_txq[tc].count = count;
1838 dev->tc_to_txq[tc].offset = offset;
1839 return 0;
1840}
1841
1842static inline
1843int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
1844{
1845 if (num_tc > TC_MAX_QUEUE)
1846 return -EINVAL;
1847
1848 dev->num_tc = num_tc;
1849 return 0;
1850}
1851
1852static inline
1853int netdev_get_num_tc(struct net_device *dev)
1854{
1855 return dev->num_tc;
1856}
1857
1858static inline
1859struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
1860 unsigned int index)
1861{
1862 return &dev->_tx[index];
1863}
1864
1865static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
1866 const struct sk_buff *skb)
1867{
1868 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
1869}
1870
1871static inline void netdev_for_each_tx_queue(struct net_device *dev,
1872 void (*f)(struct net_device *,
1873 struct netdev_queue *,
1874 void *),
1875 void *arg)
1876{
1877 unsigned int i;
1878
1879 for (i = 0; i < dev->num_tx_queues; i++)
1880 f(dev, &dev->_tx[i], arg);
1881}
1882
1883struct netdev_queue *netdev_pick_tx(struct net_device *dev,
1884 struct sk_buff *skb,
1885 void *accel_priv);
1886
1887
1888
1889
1890static inline
1891struct net *dev_net(const struct net_device *dev)
1892{
1893 return read_pnet(&dev->nd_net);
1894}
1895
1896static inline
1897void dev_net_set(struct net_device *dev, struct net *net)
1898{
1899 write_pnet(&dev->nd_net, net);
1900}
1901
1902static inline bool netdev_uses_dsa(struct net_device *dev)
1903{
1904#if IS_ENABLED(CONFIG_NET_DSA)
1905 if (dev->dsa_ptr != NULL)
1906 return dsa_uses_tagged_protocol(dev->dsa_ptr);
1907#endif
1908 return false;
1909}
1910
1911
1912
1913
1914
1915
1916
1917static inline void *netdev_priv(const struct net_device *dev)
1918{
1919 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
1920}
1921
1922
1923
1924
1925#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
1926
1927
1928
1929
1930
1931#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
1932
1933
1934
1935
1936#define NAPI_POLL_WEIGHT 64
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
1949 int (*poll)(struct napi_struct *, int), int weight);
1950
1951
1952
1953
1954
1955
1956
1957void netif_napi_del(struct napi_struct *napi);
1958
1959struct napi_gro_cb {
1960
1961 void *frag0;
1962
1963
1964 unsigned int frag0_len;
1965
1966
1967 int data_offset;
1968
1969
1970 u16 flush;
1971
1972
1973 u16 flush_id;
1974
1975
1976 u16 count;
1977
1978
1979 u16 gro_remcsum_start;
1980
1981
1982 unsigned long age;
1983
1984
1985 u16 proto;
1986
1987
1988 u8 same_flow:1;
1989
1990
1991 u8 udp_mark:1;
1992
1993
1994 u8 csum_valid:1;
1995
1996
1997 u8 csum_cnt:3;
1998
1999
2000 u8 free:2;
2001#define NAPI_GRO_FREE 1
2002#define NAPI_GRO_FREE_STOLEN_HEAD 2
2003
2004
2005 u8 is_ipv6:1;
2006
2007
2008
2009
2010 __wsum csum;
2011
2012
2013 struct sk_buff *last;
2014};
2015
2016#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
2017
2018struct packet_type {
2019 __be16 type;
2020 struct net_device *dev;
2021 int (*func) (struct sk_buff *,
2022 struct net_device *,
2023 struct packet_type *,
2024 struct net_device *);
2025 bool (*id_match)(struct packet_type *ptype,
2026 struct sock *sk);
2027 void *af_packet_priv;
2028 struct list_head list;
2029};
2030
2031struct offload_callbacks {
2032 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2033 netdev_features_t features);
2034 struct sk_buff **(*gro_receive)(struct sk_buff **head,
2035 struct sk_buff *skb);
2036 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2037};
2038
2039struct packet_offload {
2040 __be16 type;
2041 u16 priority;
2042 struct offload_callbacks callbacks;
2043 struct list_head list;
2044};
2045
2046struct udp_offload;
2047
2048struct udp_offload_callbacks {
2049 struct sk_buff **(*gro_receive)(struct sk_buff **head,
2050 struct sk_buff *skb,
2051 struct udp_offload *uoff);
2052 int (*gro_complete)(struct sk_buff *skb,
2053 int nhoff,
2054 struct udp_offload *uoff);
2055};
2056
2057struct udp_offload {
2058 __be16 port;
2059 u8 ipproto;
2060 struct udp_offload_callbacks callbacks;
2061};
2062
2063
2064struct pcpu_sw_netstats {
2065 u64 rx_packets;
2066 u64 rx_bytes;
2067 u64 tx_packets;
2068 u64 tx_bytes;
2069 struct u64_stats_sync syncp;
2070};
2071
2072#define __netdev_alloc_pcpu_stats(type, gfp) \
2073({ \
2074 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2075 if (pcpu_stats) { \
2076 int __cpu; \
2077 for_each_possible_cpu(__cpu) { \
2078 typeof(type) *stat; \
2079 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2080 u64_stats_init(&stat->syncp); \
2081 } \
2082 } \
2083 pcpu_stats; \
2084})
2085
2086#define netdev_alloc_pcpu_stats(type) \
2087 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2088
2089#include <linux/notifier.h>
2090
2091
2092
2093
2094
2095#define NETDEV_UP 0x0001
2096#define NETDEV_DOWN 0x0002
2097#define NETDEV_REBOOT 0x0003
2098
2099
2100
2101#define NETDEV_CHANGE 0x0004
2102#define NETDEV_REGISTER 0x0005
2103#define NETDEV_UNREGISTER 0x0006
2104#define NETDEV_CHANGEMTU 0x0007
2105#define NETDEV_CHANGEADDR 0x0008
2106#define NETDEV_GOING_DOWN 0x0009
2107#define NETDEV_CHANGENAME 0x000A
2108#define NETDEV_FEAT_CHANGE 0x000B
2109#define NETDEV_BONDING_FAILOVER 0x000C
2110#define NETDEV_PRE_UP 0x000D
2111#define NETDEV_PRE_TYPE_CHANGE 0x000E
2112#define NETDEV_POST_TYPE_CHANGE 0x000F
2113#define NETDEV_POST_INIT 0x0010
2114#define NETDEV_UNREGISTER_FINAL 0x0011
2115#define NETDEV_RELEASE 0x0012
2116#define NETDEV_NOTIFY_PEERS 0x0013
2117#define NETDEV_JOIN 0x0014
2118#define NETDEV_CHANGEUPPER 0x0015
2119#define NETDEV_RESEND_IGMP 0x0016
2120#define NETDEV_PRECHANGEMTU 0x0017
2121#define NETDEV_CHANGEINFODATA 0x0018
2122#define NETDEV_BONDING_INFO 0x0019
2123#define NETDEV_PRECHANGEUPPER 0x001A
2124
2125int register_netdevice_notifier(struct notifier_block *nb);
2126int unregister_netdevice_notifier(struct notifier_block *nb);
2127
2128struct netdev_notifier_info {
2129 struct net_device *dev;
2130};
2131
2132struct netdev_notifier_change_info {
2133 struct netdev_notifier_info info;
2134 unsigned int flags_changed;
2135};
2136
2137struct netdev_notifier_changeupper_info {
2138 struct netdev_notifier_info info;
2139 struct net_device *upper_dev;
2140 bool master;
2141 bool linking;
2142};
2143
2144static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2145 struct net_device *dev)
2146{
2147 info->dev = dev;
2148}
2149
2150static inline struct net_device *
2151netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2152{
2153 return info->dev;
2154}
2155
2156int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2157
2158
2159extern rwlock_t dev_base_lock;
2160
2161#define for_each_netdev(net, d) \
2162 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2163#define for_each_netdev_reverse(net, d) \
2164 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2165#define for_each_netdev_rcu(net, d) \
2166 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2167#define for_each_netdev_safe(net, d, n) \
2168 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2169#define for_each_netdev_continue(net, d) \
2170 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2171#define for_each_netdev_continue_rcu(net, d) \
2172 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2173#define for_each_netdev_in_bond_rcu(bond, slave) \
2174 for_each_netdev_rcu(&init_net, slave) \
2175 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2176#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
2177
2178static inline struct net_device *next_net_device(struct net_device *dev)
2179{
2180 struct list_head *lh;
2181 struct net *net;
2182
2183 net = dev_net(dev);
2184 lh = dev->dev_list.next;
2185 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2186}
2187
2188static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2189{
2190 struct list_head *lh;
2191 struct net *net;
2192
2193 net = dev_net(dev);
2194 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2195 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2196}
2197
2198static inline struct net_device *first_net_device(struct net *net)
2199{
2200 return list_empty(&net->dev_base_head) ? NULL :
2201 net_device_entry(net->dev_base_head.next);
2202}
2203
2204static inline struct net_device *first_net_device_rcu(struct net *net)
2205{
2206 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2207
2208 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2209}
2210
2211int netdev_boot_setup_check(struct net_device *dev);
2212unsigned long netdev_boot_base(const char *prefix, int unit);
2213struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2214 const char *hwaddr);
2215struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2216struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
2217void dev_add_pack(struct packet_type *pt);
2218void dev_remove_pack(struct packet_type *pt);
2219void __dev_remove_pack(struct packet_type *pt);
2220void dev_add_offload(struct packet_offload *po);
2221void dev_remove_offload(struct packet_offload *po);
2222
2223int dev_get_iflink(const struct net_device *dev);
2224int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
2225struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2226 unsigned short mask);
2227struct net_device *dev_get_by_name(struct net *net, const char *name);
2228struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2229struct net_device *__dev_get_by_name(struct net *net, const char *name);
2230int dev_alloc_name(struct net_device *dev, const char *name);
2231int dev_open(struct net_device *dev);
2232int dev_close(struct net_device *dev);
2233int dev_close_many(struct list_head *head, bool unlink);
2234void dev_disable_lro(struct net_device *dev);
2235int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
2236int dev_queue_xmit(struct sk_buff *skb);
2237int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
2238int register_netdevice(struct net_device *dev);
2239void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
2240void unregister_netdevice_many(struct list_head *head);
2241static inline void unregister_netdevice(struct net_device *dev)
2242{
2243 unregister_netdevice_queue(dev, NULL);
2244}
2245
2246int netdev_refcnt_read(const struct net_device *dev);
2247void free_netdev(struct net_device *dev);
2248void netdev_freemem(struct net_device *dev);
2249void synchronize_net(void);
2250int init_dummy_netdev(struct net_device *dev);
2251
2252DECLARE_PER_CPU(int, xmit_recursion);
2253static inline int dev_recursion_level(void)
2254{
2255 return this_cpu_read(xmit_recursion);
2256}
2257
2258struct net_device *dev_get_by_index(struct net *net, int ifindex);
2259struct net_device *__dev_get_by_index(struct net *net, int ifindex);
2260struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
2261int netdev_get_name(struct net *net, char *name, int ifindex);
2262int dev_restart(struct net_device *dev);
2263int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2264
2265static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
2266{
2267 return NAPI_GRO_CB(skb)->data_offset;
2268}
2269
2270static inline unsigned int skb_gro_len(const struct sk_buff *skb)
2271{
2272 return skb->len - NAPI_GRO_CB(skb)->data_offset;
2273}
2274
2275static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
2276{
2277 NAPI_GRO_CB(skb)->data_offset += len;
2278}
2279
2280static inline void *skb_gro_header_fast(struct sk_buff *skb,
2281 unsigned int offset)
2282{
2283 return NAPI_GRO_CB(skb)->frag0 + offset;
2284}
2285
2286static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
2287{
2288 return NAPI_GRO_CB(skb)->frag0_len < hlen;
2289}
2290
2291static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
2292 unsigned int offset)
2293{
2294 if (!pskb_may_pull(skb, hlen))
2295 return NULL;
2296
2297 NAPI_GRO_CB(skb)->frag0 = NULL;
2298 NAPI_GRO_CB(skb)->frag0_len = 0;
2299 return skb->data + offset;
2300}
2301
2302static inline void *skb_gro_network_header(struct sk_buff *skb)
2303{
2304 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
2305 skb_network_offset(skb);
2306}
2307
2308static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
2309 const void *start, unsigned int len)
2310{
2311 if (NAPI_GRO_CB(skb)->csum_valid)
2312 NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
2313 csum_partial(start, len, 0));
2314}
2315
2316
2317
2318
2319
2320
2321__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
2322
2323static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
2324{
2325 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
2326}
2327
2328static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
2329 bool zero_okay,
2330 __sum16 check)
2331{
2332 return ((skb->ip_summed != CHECKSUM_PARTIAL ||
2333 skb_checksum_start_offset(skb) <
2334 skb_gro_offset(skb)) &&
2335 !skb_at_gro_remcsum_start(skb) &&
2336 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2337 (!zero_okay || check));
2338}
2339
2340static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
2341 __wsum psum)
2342{
2343 if (NAPI_GRO_CB(skb)->csum_valid &&
2344 !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
2345 return 0;
2346
2347 NAPI_GRO_CB(skb)->csum = psum;
2348
2349 return __skb_gro_checksum_complete(skb);
2350}
2351
2352static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
2353{
2354 if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
2355
2356 NAPI_GRO_CB(skb)->csum_cnt--;
2357 } else {
2358
2359
2360
2361
2362 __skb_incr_checksum_unnecessary(skb);
2363 }
2364}
2365
2366#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \
2367 compute_pseudo) \
2368({ \
2369 __sum16 __ret = 0; \
2370 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \
2371 __ret = __skb_gro_checksum_validate_complete(skb, \
2372 compute_pseudo(skb, proto)); \
2373 if (__ret) \
2374 __skb_mark_checksum_bad(skb); \
2375 else \
2376 skb_gro_incr_csum_unnecessary(skb); \
2377 __ret; \
2378})
2379
2380#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \
2381 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
2382
2383#define skb_gro_checksum_validate_zero_check(skb, proto, check, \
2384 compute_pseudo) \
2385 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
2386
2387#define skb_gro_checksum_simple_validate(skb) \
2388 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
2389
2390static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
2391{
2392 return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
2393 !NAPI_GRO_CB(skb)->csum_valid);
2394}
2395
2396static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2397 __sum16 check, __wsum pseudo)
2398{
2399 NAPI_GRO_CB(skb)->csum = ~pseudo;
2400 NAPI_GRO_CB(skb)->csum_valid = 1;
2401}
2402
2403#define skb_gro_checksum_try_convert(skb, proto, check, compute_pseudo) \
2404do { \
2405 if (__skb_gro_checksum_convert_check(skb)) \
2406 __skb_gro_checksum_convert(skb, check, \
2407 compute_pseudo(skb, proto)); \
2408} while (0)
2409
2410struct gro_remcsum {
2411 int offset;
2412 __wsum delta;
2413};
2414
2415static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
2416{
2417 grc->offset = 0;
2418 grc->delta = 0;
2419}
2420
2421static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
2422 unsigned int off, size_t hdrlen,
2423 int start, int offset,
2424 struct gro_remcsum *grc,
2425 bool nopartial)
2426{
2427 __wsum delta;
2428 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
2429
2430 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
2431
2432 if (!nopartial) {
2433 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
2434 return ptr;
2435 }
2436
2437 ptr = skb_gro_header_fast(skb, off);
2438 if (skb_gro_header_hard(skb, off + plen)) {
2439 ptr = skb_gro_header_slow(skb, off + plen, off);
2440 if (!ptr)
2441 return NULL;
2442 }
2443
2444 delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
2445 start, offset);
2446
2447
2448 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
2449
2450 grc->offset = off + hdrlen + offset;
2451 grc->delta = delta;
2452
2453 return ptr;
2454}
2455
2456static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
2457 struct gro_remcsum *grc)
2458{
2459 void *ptr;
2460 size_t plen = grc->offset + sizeof(u16);
2461
2462 if (!grc->delta)
2463 return;
2464
2465 ptr = skb_gro_header_fast(skb, grc->offset);
2466 if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
2467 ptr = skb_gro_header_slow(skb, plen, grc->offset);
2468 if (!ptr)
2469 return;
2470 }
2471
2472 remcsum_unadjust((__sum16 *)ptr, grc->delta);
2473}
2474
2475static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
2476 unsigned short type,
2477 const void *daddr, const void *saddr,
2478 unsigned int len)
2479{
2480 if (!dev->header_ops || !dev->header_ops->create)
2481 return 0;
2482
2483 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
2484}
2485
2486static inline int dev_parse_header(const struct sk_buff *skb,
2487 unsigned char *haddr)
2488{
2489 const struct net_device *dev = skb->dev;
2490
2491 if (!dev->header_ops || !dev->header_ops->parse)
2492 return 0;
2493 return dev->header_ops->parse(skb, haddr);
2494}
2495
2496typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr, int len);
2497int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
2498static inline int unregister_gifconf(unsigned int family)
2499{
2500 return register_gifconf(family, NULL);
2501}
2502
2503#ifdef CONFIG_NET_FLOW_LIMIT
2504#define FLOW_LIMIT_HISTORY (1 << 7)
2505struct sd_flow_limit {
2506 u64 count;
2507 unsigned int num_buckets;
2508 unsigned int history_head;
2509 u16 history[FLOW_LIMIT_HISTORY];
2510 u8 buckets[];
2511};
2512
2513extern int netdev_flow_limit_table_len;
2514#endif
2515
2516
2517
2518
2519struct softnet_data {
2520 struct list_head poll_list;
2521 struct sk_buff_head process_queue;
2522
2523
2524 unsigned int processed;
2525 unsigned int time_squeeze;
2526 unsigned int cpu_collision;
2527 unsigned int received_rps;
2528#ifdef CONFIG_RPS
2529 struct softnet_data *rps_ipi_list;
2530#endif
2531#ifdef CONFIG_NET_FLOW_LIMIT
2532 struct sd_flow_limit __rcu *flow_limit;
2533#endif
2534 struct Qdisc *output_queue;
2535 struct Qdisc **output_queue_tailp;
2536 struct sk_buff *completion_queue;
2537
2538#ifdef CONFIG_RPS
2539
2540 struct call_single_data csd ____cacheline_aligned_in_smp;
2541 struct softnet_data *rps_ipi_next;
2542 unsigned int cpu;
2543 unsigned int input_queue_head;
2544 unsigned int input_queue_tail;
2545#endif
2546 unsigned int dropped;
2547 struct sk_buff_head input_pkt_queue;
2548 struct napi_struct backlog;
2549
2550};
2551
2552static inline void input_queue_head_incr(struct softnet_data *sd)
2553{
2554#ifdef CONFIG_RPS
2555 sd->input_queue_head++;
2556#endif
2557}
2558
2559static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2560 unsigned int *qtail)
2561{
2562#ifdef CONFIG_RPS
2563 *qtail = ++sd->input_queue_tail;
2564#endif
2565}
2566
2567DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
2568
2569void __netif_schedule(struct Qdisc *q);
2570void netif_schedule_queue(struct netdev_queue *txq);
2571
2572static inline void netif_tx_schedule_all(struct net_device *dev)
2573{
2574 unsigned int i;
2575
2576 for (i = 0; i < dev->num_tx_queues; i++)
2577 netif_schedule_queue(netdev_get_tx_queue(dev, i));
2578}
2579
2580static inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
2581{
2582 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2583}
2584
2585
2586
2587
2588
2589
2590
2591static inline void netif_start_queue(struct net_device *dev)
2592{
2593 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
2594}
2595
2596static inline void netif_tx_start_all_queues(struct net_device *dev)
2597{
2598 unsigned int i;
2599
2600 for (i = 0; i < dev->num_tx_queues; i++) {
2601 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2602 netif_tx_start_queue(txq);
2603 }
2604}
2605
2606void netif_tx_wake_queue(struct netdev_queue *dev_queue);
2607
2608
2609
2610
2611
2612
2613
2614
2615static inline void netif_wake_queue(struct net_device *dev)
2616{
2617 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
2618}
2619
2620static inline void netif_tx_wake_all_queues(struct net_device *dev)
2621{
2622 unsigned int i;
2623
2624 for (i = 0; i < dev->num_tx_queues; i++) {
2625 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2626 netif_tx_wake_queue(txq);
2627 }
2628}
2629
2630static inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
2631{
2632 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2633}
2634
2635
2636
2637
2638
2639
2640
2641
2642static inline void netif_stop_queue(struct net_device *dev)
2643{
2644 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
2645}
2646
2647void netif_tx_stop_all_queues(struct net_device *dev);
2648
2649static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
2650{
2651 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2652}
2653
2654
2655
2656
2657
2658
2659
2660static inline bool netif_queue_stopped(const struct net_device *dev)
2661{
2662 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
2663}
2664
2665static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
2666{
2667 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2668}
2669
2670static inline bool
2671netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
2672{
2673 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2674}
2675
2676static inline bool
2677netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
2678{
2679 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
2680}
2681
2682
2683
2684
2685
2686
2687
2688
2689static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
2690{
2691#ifdef CONFIG_BQL
2692 prefetchw(&dev_queue->dql.num_queued);
2693#endif
2694}
2695
2696
2697
2698
2699
2700
2701
2702
2703static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
2704{
2705#ifdef CONFIG_BQL
2706 prefetchw(&dev_queue->dql.limit);
2707#endif
2708}
2709
2710static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
2711 unsigned int bytes)
2712{
2713#ifdef CONFIG_BQL
2714 dql_queued(&dev_queue->dql, bytes);
2715
2716 if (likely(dql_avail(&dev_queue->dql) >= 0))
2717 return;
2718
2719 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2720
2721
2722
2723
2724
2725
2726 smp_mb();
2727
2728
2729 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
2730 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2731#endif
2732}
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
2744{
2745 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
2746}
2747
2748static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
2749 unsigned int pkts, unsigned int bytes)
2750{
2751#ifdef CONFIG_BQL
2752 if (unlikely(!bytes))
2753 return;
2754
2755 dql_completed(&dev_queue->dql, bytes);
2756
2757
2758
2759
2760
2761
2762 smp_mb();
2763
2764 if (dql_avail(&dev_queue->dql) < 0)
2765 return;
2766
2767 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
2768 netif_schedule_queue(dev_queue);
2769#endif
2770}
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782static inline void netdev_completed_queue(struct net_device *dev,
2783 unsigned int pkts, unsigned int bytes)
2784{
2785 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
2786}
2787
2788static inline void netdev_tx_reset_queue(struct netdev_queue *q)
2789{
2790#ifdef CONFIG_BQL
2791 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
2792 dql_reset(&q->dql);
2793#endif
2794}
2795
2796
2797
2798
2799
2800
2801
2802
2803static inline void netdev_reset_queue(struct net_device *dev_queue)
2804{
2805 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
2806}
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
2817{
2818 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2819 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2820 dev->name, queue_index,
2821 dev->real_num_tx_queues);
2822 return 0;
2823 }
2824
2825 return queue_index;
2826}
2827
2828
2829
2830
2831
2832
2833
2834static inline bool netif_running(const struct net_device *dev)
2835{
2836 return test_bit(__LINK_STATE_START, &dev->state);
2837}
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
2854{
2855 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2856
2857 netif_tx_start_queue(txq);
2858}
2859
2860
2861
2862
2863
2864
2865
2866
2867static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
2868{
2869 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2870 netif_tx_stop_queue(txq);
2871}
2872
2873
2874
2875
2876
2877
2878
2879
2880static inline bool __netif_subqueue_stopped(const struct net_device *dev,
2881 u16 queue_index)
2882{
2883 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2884
2885 return netif_tx_queue_stopped(txq);
2886}
2887
2888static inline bool netif_subqueue_stopped(const struct net_device *dev,
2889 struct sk_buff *skb)
2890{
2891 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
2892}
2893
2894void netif_wake_subqueue(struct net_device *dev, u16 queue_index);
2895
2896#ifdef CONFIG_XPS
2897int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2898 u16 index);
2899#else
2900static inline int netif_set_xps_queue(struct net_device *dev,
2901 const struct cpumask *mask,
2902 u16 index)
2903{
2904 return 0;
2905}
2906#endif
2907
2908u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2909 unsigned int num_tx_queues);
2910
2911
2912
2913
2914
2915static inline u16 skb_tx_hash(const struct net_device *dev,
2916 struct sk_buff *skb)
2917{
2918 return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
2919}
2920
2921
2922
2923
2924
2925
2926
2927static inline bool netif_is_multiqueue(const struct net_device *dev)
2928{
2929 return dev->num_tx_queues > 1;
2930}
2931
2932int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
2933
2934#ifdef CONFIG_SYSFS
2935int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
2936#else
2937static inline int netif_set_real_num_rx_queues(struct net_device *dev,
2938 unsigned int rxq)
2939{
2940 return 0;
2941}
2942#endif
2943
2944#ifdef CONFIG_SYSFS
2945static inline unsigned int get_netdev_rx_queue_index(
2946 struct netdev_rx_queue *queue)
2947{
2948 struct net_device *dev = queue->dev;
2949 int index = queue - dev->_rx;
2950
2951 BUG_ON(index >= dev->num_rx_queues);
2952 return index;
2953}
2954#endif
2955
2956#define DEFAULT_MAX_NUM_RSS_QUEUES (8)
2957int netif_get_num_default_rss_queues(void);
2958
2959enum skb_free_reason {
2960 SKB_REASON_CONSUMED,
2961 SKB_REASON_DROPPED,
2962};
2963
2964void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
2965void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986static inline void dev_kfree_skb_irq(struct sk_buff *skb)
2987{
2988 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
2989}
2990
2991static inline void dev_consume_skb_irq(struct sk_buff *skb)
2992{
2993 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
2994}
2995
2996static inline void dev_kfree_skb_any(struct sk_buff *skb)
2997{
2998 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
2999}
3000
3001static inline void dev_consume_skb_any(struct sk_buff *skb)
3002{
3003 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3004}
3005
3006int netif_rx(struct sk_buff *skb);
3007int netif_rx_ni(struct sk_buff *skb);
3008int netif_receive_skb(struct sk_buff *skb);
3009gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3010void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3011struct sk_buff *napi_get_frags(struct napi_struct *napi);
3012gro_result_t napi_gro_frags(struct napi_struct *napi);
3013struct packet_offload *gro_find_receive_by_type(__be16 type);
3014struct packet_offload *gro_find_complete_by_type(__be16 type);
3015
3016static inline void napi_free_frags(struct napi_struct *napi)
3017{
3018 kfree_skb(napi->skb);
3019 napi->skb = NULL;
3020}
3021
3022int netdev_rx_handler_register(struct net_device *dev,
3023 rx_handler_func_t *rx_handler,
3024 void *rx_handler_data);
3025void netdev_rx_handler_unregister(struct net_device *dev);
3026
3027bool dev_valid_name(const char *name);
3028int dev_ioctl(struct net *net, unsigned int cmd, void __user *);
3029int dev_ethtool(struct net *net, struct ifreq *);
3030unsigned int dev_get_flags(const struct net_device *);
3031int __dev_change_flags(struct net_device *, unsigned int flags);
3032int dev_change_flags(struct net_device *, unsigned int);
3033void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3034 unsigned int gchanges);
3035int dev_change_name(struct net_device *, const char *);
3036int dev_set_alias(struct net_device *, const char *, size_t);
3037int dev_change_net_namespace(struct net_device *, struct net *, const char *);
3038int dev_set_mtu(struct net_device *, int);
3039void dev_set_group(struct net_device *, int);
3040int dev_set_mac_address(struct net_device *, struct sockaddr *);
3041int dev_change_carrier(struct net_device *, bool new_carrier);
3042int dev_get_phys_port_id(struct net_device *dev,
3043 struct netdev_phys_item_id *ppid);
3044int dev_get_phys_port_name(struct net_device *dev,
3045 char *name, size_t len);
3046int dev_change_proto_down(struct net_device *dev, bool proto_down);
3047struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev);
3048struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3049 struct netdev_queue *txq, int *ret);
3050int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3051int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3052bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb);
3053
3054extern int netdev_budget;
3055
3056
3057void netdev_run_todo(void);
3058
3059
3060
3061
3062
3063
3064
3065static inline void dev_put(struct net_device *dev)
3066{
3067 this_cpu_dec(*dev->pcpu_refcnt);
3068}
3069
3070
3071
3072
3073
3074
3075
3076static inline void dev_hold(struct net_device *dev)
3077{
3078 this_cpu_inc(*dev->pcpu_refcnt);
3079}
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090void linkwatch_init_dev(struct net_device *dev);
3091void linkwatch_fire_event(struct net_device *dev);
3092void linkwatch_forget_dev(struct net_device *dev);
3093
3094
3095
3096
3097
3098
3099
3100static inline bool netif_carrier_ok(const struct net_device *dev)
3101{
3102 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
3103}
3104
3105unsigned long dev_trans_start(struct net_device *dev);
3106
3107void __netdev_watchdog_up(struct net_device *dev);
3108
3109void netif_carrier_on(struct net_device *dev);
3110
3111void netif_carrier_off(struct net_device *dev);
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126static inline void netif_dormant_on(struct net_device *dev)
3127{
3128 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
3129 linkwatch_fire_event(dev);
3130}
3131
3132
3133
3134
3135
3136
3137
3138static inline void netif_dormant_off(struct net_device *dev)
3139{
3140 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
3141 linkwatch_fire_event(dev);
3142}
3143
3144
3145
3146
3147
3148
3149
3150static inline bool netif_dormant(const struct net_device *dev)
3151{
3152 return test_bit(__LINK_STATE_DORMANT, &dev->state);
3153}
3154
3155
3156
3157
3158
3159
3160
3161
3162static inline bool netif_oper_up(const struct net_device *dev)
3163{
3164 return (dev->operstate == IF_OPER_UP ||
3165 dev->operstate == IF_OPER_UNKNOWN );
3166}
3167
3168
3169
3170
3171
3172
3173
3174static inline bool netif_device_present(struct net_device *dev)
3175{
3176 return test_bit(__LINK_STATE_PRESENT, &dev->state);
3177}
3178
3179void netif_device_detach(struct net_device *dev);
3180
3181void netif_device_attach(struct net_device *dev);
3182
3183
3184
3185
3186
3187enum {
3188 NETIF_MSG_DRV = 0x0001,
3189 NETIF_MSG_PROBE = 0x0002,
3190 NETIF_MSG_LINK = 0x0004,
3191 NETIF_MSG_TIMER = 0x0008,
3192 NETIF_MSG_IFDOWN = 0x0010,
3193 NETIF_MSG_IFUP = 0x0020,
3194 NETIF_MSG_RX_ERR = 0x0040,
3195 NETIF_MSG_TX_ERR = 0x0080,
3196 NETIF_MSG_TX_QUEUED = 0x0100,
3197 NETIF_MSG_INTR = 0x0200,
3198 NETIF_MSG_TX_DONE = 0x0400,
3199 NETIF_MSG_RX_STATUS = 0x0800,
3200 NETIF_MSG_PKTDATA = 0x1000,
3201 NETIF_MSG_HW = 0x2000,
3202 NETIF_MSG_WOL = 0x4000,
3203};
3204
3205#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
3206#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
3207#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
3208#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
3209#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
3210#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
3211#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
3212#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
3213#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
3214#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
3215#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
3216#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
3217#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
3218#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
3219#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
3220
3221static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
3222{
3223
3224 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
3225 return default_msg_enable_bits;
3226 if (debug_value == 0)
3227 return 0;
3228
3229 return (1 << debug_value) - 1;
3230}
3231
3232static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
3233{
3234 spin_lock(&txq->_xmit_lock);
3235 txq->xmit_lock_owner = cpu;
3236}
3237
3238static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
3239{
3240 spin_lock_bh(&txq->_xmit_lock);
3241 txq->xmit_lock_owner = smp_processor_id();
3242}
3243
3244static inline bool __netif_tx_trylock(struct netdev_queue *txq)
3245{
3246 bool ok = spin_trylock(&txq->_xmit_lock);
3247 if (likely(ok))
3248 txq->xmit_lock_owner = smp_processor_id();
3249 return ok;
3250}
3251
3252static inline void __netif_tx_unlock(struct netdev_queue *txq)
3253{
3254 txq->xmit_lock_owner = -1;
3255 spin_unlock(&txq->_xmit_lock);
3256}
3257
3258static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
3259{
3260 txq->xmit_lock_owner = -1;
3261 spin_unlock_bh(&txq->_xmit_lock);
3262}
3263
3264static inline void txq_trans_update(struct netdev_queue *txq)
3265{
3266 if (txq->xmit_lock_owner != -1)
3267 txq->trans_start = jiffies;
3268}
3269
3270
3271
3272
3273
3274
3275
3276static inline void netif_tx_lock(struct net_device *dev)
3277{
3278 unsigned int i;
3279 int cpu;
3280
3281 spin_lock(&dev->tx_global_lock);
3282 cpu = smp_processor_id();
3283 for (i = 0; i < dev->num_tx_queues; i++) {
3284 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3285
3286
3287
3288
3289
3290
3291
3292 __netif_tx_lock(txq, cpu);
3293 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
3294 __netif_tx_unlock(txq);
3295 }
3296}
3297
3298static inline void netif_tx_lock_bh(struct net_device *dev)
3299{
3300 local_bh_disable();
3301 netif_tx_lock(dev);
3302}
3303
3304static inline void netif_tx_unlock(struct net_device *dev)
3305{
3306 unsigned int i;
3307
3308 for (i = 0; i < dev->num_tx_queues; i++) {
3309 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3310
3311
3312
3313
3314
3315 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
3316 netif_schedule_queue(txq);
3317 }
3318 spin_unlock(&dev->tx_global_lock);
3319}
3320
3321static inline void netif_tx_unlock_bh(struct net_device *dev)
3322{
3323 netif_tx_unlock(dev);
3324 local_bh_enable();
3325}
3326
3327#define HARD_TX_LOCK(dev, txq, cpu) { \
3328 if ((dev->features & NETIF_F_LLTX) == 0) { \
3329 __netif_tx_lock(txq, cpu); \
3330 } \
3331}
3332
3333#define HARD_TX_TRYLOCK(dev, txq) \
3334 (((dev->features & NETIF_F_LLTX) == 0) ? \
3335 __netif_tx_trylock(txq) : \
3336 true )
3337
3338#define HARD_TX_UNLOCK(dev, txq) { \
3339 if ((dev->features & NETIF_F_LLTX) == 0) { \
3340 __netif_tx_unlock(txq); \
3341 } \
3342}
3343
3344static inline void netif_tx_disable(struct net_device *dev)
3345{
3346 unsigned int i;
3347 int cpu;
3348
3349 local_bh_disable();
3350 cpu = smp_processor_id();
3351 for (i = 0; i < dev->num_tx_queues; i++) {
3352 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3353
3354 __netif_tx_lock(txq, cpu);
3355 netif_tx_stop_queue(txq);
3356 __netif_tx_unlock(txq);
3357 }
3358 local_bh_enable();
3359}
3360
3361static inline void netif_addr_lock(struct net_device *dev)
3362{
3363 spin_lock(&dev->addr_list_lock);
3364}
3365
3366static inline void netif_addr_lock_nested(struct net_device *dev)
3367{
3368 int subclass = SINGLE_DEPTH_NESTING;
3369
3370 if (dev->netdev_ops->ndo_get_lock_subclass)
3371 subclass = dev->netdev_ops->ndo_get_lock_subclass(dev);
3372
3373 spin_lock_nested(&dev->addr_list_lock, subclass);
3374}
3375
3376static inline void netif_addr_lock_bh(struct net_device *dev)
3377{
3378 spin_lock_bh(&dev->addr_list_lock);
3379}
3380
3381static inline void netif_addr_unlock(struct net_device *dev)
3382{
3383 spin_unlock(&dev->addr_list_lock);
3384}
3385
3386static inline void netif_addr_unlock_bh(struct net_device *dev)
3387{
3388 spin_unlock_bh(&dev->addr_list_lock);
3389}
3390
3391
3392
3393
3394
3395#define for_each_dev_addr(dev, ha) \
3396 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
3397
3398
3399
3400void ether_setup(struct net_device *dev);
3401
3402
3403struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
3404 unsigned char name_assign_type,
3405 void (*setup)(struct net_device *),
3406 unsigned int txqs, unsigned int rxqs);
3407#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
3408 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
3409
3410#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
3411 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
3412 count)
3413
3414int register_netdev(struct net_device *dev);
3415void unregister_netdev(struct net_device *dev);
3416
3417
3418int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
3419 struct netdev_hw_addr_list *from_list, int addr_len);
3420void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
3421 struct netdev_hw_addr_list *from_list, int addr_len);
3422int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
3423 struct net_device *dev,
3424 int (*sync)(struct net_device *, const unsigned char *),
3425 int (*unsync)(struct net_device *,
3426 const unsigned char *));
3427void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
3428 struct net_device *dev,
3429 int (*unsync)(struct net_device *,
3430 const unsigned char *));
3431void __hw_addr_init(struct netdev_hw_addr_list *list);
3432
3433
3434int dev_addr_add(struct net_device *dev, const unsigned char *addr,
3435 unsigned char addr_type);
3436int dev_addr_del(struct net_device *dev, const unsigned char *addr,
3437 unsigned char addr_type);
3438void dev_addr_flush(struct net_device *dev);
3439int dev_addr_init(struct net_device *dev);
3440
3441
3442int dev_uc_add(struct net_device *dev, const unsigned char *addr);
3443int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
3444int dev_uc_del(struct net_device *dev, const unsigned char *addr);
3445int dev_uc_sync(struct net_device *to, struct net_device *from);
3446int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
3447void dev_uc_unsync(struct net_device *to, struct net_device *from);
3448void dev_uc_flush(struct net_device *dev);
3449void dev_uc_init(struct net_device *dev);
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460static inline int __dev_uc_sync(struct net_device *dev,
3461 int (*sync)(struct net_device *,
3462 const unsigned char *),
3463 int (*unsync)(struct net_device *,
3464 const unsigned char *))
3465{
3466 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
3467}
3468
3469
3470
3471
3472
3473
3474
3475
3476static inline void __dev_uc_unsync(struct net_device *dev,
3477 int (*unsync)(struct net_device *,
3478 const unsigned char *))
3479{
3480 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
3481}
3482
3483
3484int dev_mc_add(struct net_device *dev, const unsigned char *addr);
3485int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
3486int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
3487int dev_mc_del(struct net_device *dev, const unsigned char *addr);
3488int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
3489int dev_mc_sync(struct net_device *to, struct net_device *from);
3490int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
3491void dev_mc_unsync(struct net_device *to, struct net_device *from);
3492void dev_mc_flush(struct net_device *dev);
3493void dev_mc_init(struct net_device *dev);
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504static inline int __dev_mc_sync(struct net_device *dev,
3505 int (*sync)(struct net_device *,
3506 const unsigned char *),
3507 int (*unsync)(struct net_device *,
3508 const unsigned char *))
3509{
3510 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
3511}
3512
3513
3514
3515
3516
3517
3518
3519
3520static inline void __dev_mc_unsync(struct net_device *dev,
3521 int (*unsync)(struct net_device *,
3522 const unsigned char *))
3523{
3524 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
3525}
3526
3527
3528void dev_set_rx_mode(struct net_device *dev);
3529void __dev_set_rx_mode(struct net_device *dev);
3530int dev_set_promiscuity(struct net_device *dev, int inc);
3531int dev_set_allmulti(struct net_device *dev, int inc);
3532void netdev_state_change(struct net_device *dev);
3533void netdev_notify_peers(struct net_device *dev);
3534void netdev_features_change(struct net_device *dev);
3535
3536void dev_load(struct net *net, const char *name);
3537struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
3538 struct rtnl_link_stats64 *storage);
3539void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
3540 const struct net_device_stats *netdev_stats);
3541
3542extern int netdev_max_backlog;
3543extern int netdev_tstamp_prequeue;
3544extern int weight_p;
3545extern int bpf_jit_enable;
3546
3547bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
3548struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
3549 struct list_head **iter);
3550struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
3551 struct list_head **iter);
3552
3553
3554#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
3555 for (iter = &(dev)->adj_list.upper, \
3556 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
3557 updev; \
3558 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
3559
3560
3561#define netdev_for_each_all_upper_dev_rcu(dev, updev, iter) \
3562 for (iter = &(dev)->all_adj_list.upper, \
3563 updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)); \
3564 updev; \
3565 updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)))
3566
3567void *netdev_lower_get_next_private(struct net_device *dev,
3568 struct list_head **iter);
3569void *netdev_lower_get_next_private_rcu(struct net_device *dev,
3570 struct list_head **iter);
3571
3572#define netdev_for_each_lower_private(dev, priv, iter) \
3573 for (iter = (dev)->adj_list.lower.next, \
3574 priv = netdev_lower_get_next_private(dev, &(iter)); \
3575 priv; \
3576 priv = netdev_lower_get_next_private(dev, &(iter)))
3577
3578#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
3579 for (iter = &(dev)->adj_list.lower, \
3580 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
3581 priv; \
3582 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
3583
3584void *netdev_lower_get_next(struct net_device *dev,
3585 struct list_head **iter);
3586#define netdev_for_each_lower_dev(dev, ldev, iter) \
3587 for (iter = &(dev)->adj_list.lower, \
3588 ldev = netdev_lower_get_next(dev, &(iter)); \
3589 ldev; \
3590 ldev = netdev_lower_get_next(dev, &(iter)))
3591
3592void *netdev_adjacent_get_private(struct list_head *adj_list);
3593void *netdev_lower_get_first_private_rcu(struct net_device *dev);
3594struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
3595struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
3596int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev);
3597int netdev_master_upper_dev_link(struct net_device *dev,
3598 struct net_device *upper_dev);
3599int netdev_master_upper_dev_link_private(struct net_device *dev,
3600 struct net_device *upper_dev,
3601 void *private);
3602void netdev_upper_dev_unlink(struct net_device *dev,
3603 struct net_device *upper_dev);
3604void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
3605void *netdev_lower_dev_get_private(struct net_device *dev,
3606 struct net_device *lower_dev);
3607
3608
3609#define NETDEV_RSS_KEY_LEN 52
3610extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN];
3611void netdev_rss_key_fill(void *buffer, size_t len);
3612
3613int dev_get_nest_level(struct net_device *dev,
3614 bool (*type_check)(struct net_device *dev));
3615int skb_checksum_help(struct sk_buff *skb);
3616struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3617 netdev_features_t features, bool tx_path);
3618struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3619 netdev_features_t features);
3620
3621struct netdev_bonding_info {
3622 ifslave slave;
3623 ifbond master;
3624};
3625
3626struct netdev_notifier_bonding_info {
3627 struct netdev_notifier_info info;
3628 struct netdev_bonding_info bonding_info;
3629};
3630
3631void netdev_bonding_info_change(struct net_device *dev,
3632 struct netdev_bonding_info *bonding_info);
3633
3634static inline
3635struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
3636{
3637 return __skb_gso_segment(skb, features, true);
3638}
3639__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
3640
3641static inline bool can_checksum_protocol(netdev_features_t features,
3642 __be16 protocol)
3643{
3644 return ((features & NETIF_F_GEN_CSUM) ||
3645 ((features & NETIF_F_V4_CSUM) &&
3646 protocol == htons(ETH_P_IP)) ||
3647 ((features & NETIF_F_V6_CSUM) &&
3648 protocol == htons(ETH_P_IPV6)) ||
3649 ((features & NETIF_F_FCOE_CRC) &&
3650 protocol == htons(ETH_P_FCOE)));
3651}
3652
3653#ifdef CONFIG_BUG
3654void netdev_rx_csum_fault(struct net_device *dev);
3655#else
3656static inline void netdev_rx_csum_fault(struct net_device *dev)
3657{
3658}
3659#endif
3660
3661void net_enable_timestamp(void);
3662void net_disable_timestamp(void);
3663
3664#ifdef CONFIG_PROC_FS
3665int __init dev_proc_init(void);
3666#else
3667#define dev_proc_init() 0
3668#endif
3669
3670static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
3671 struct sk_buff *skb, struct net_device *dev,
3672 bool more)
3673{
3674 skb->xmit_more = more ? 1 : 0;
3675 return ops->ndo_start_xmit(skb, dev);
3676}
3677
3678static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
3679 struct netdev_queue *txq, bool more)
3680{
3681 const struct net_device_ops *ops = dev->netdev_ops;
3682 int rc;
3683
3684 rc = __netdev_start_xmit(ops, skb, dev, more);
3685 if (rc == NETDEV_TX_OK)
3686 txq_trans_update(txq);
3687
3688 return rc;
3689}
3690
3691int netdev_class_create_file_ns(struct class_attribute *class_attr,
3692 const void *ns);
3693void netdev_class_remove_file_ns(struct class_attribute *class_attr,
3694 const void *ns);
3695
3696static inline int netdev_class_create_file(struct class_attribute *class_attr)
3697{
3698 return netdev_class_create_file_ns(class_attr, NULL);
3699}
3700
3701static inline void netdev_class_remove_file(struct class_attribute *class_attr)
3702{
3703 netdev_class_remove_file_ns(class_attr, NULL);
3704}
3705
3706extern struct kobj_ns_type_operations net_ns_type_operations;
3707
3708const char *netdev_drivername(const struct net_device *dev);
3709
3710void linkwatch_run_queue(void);
3711
3712static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
3713 netdev_features_t f2)
3714{
3715 if (f1 & NETIF_F_GEN_CSUM)
3716 f1 |= (NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3717 if (f2 & NETIF_F_GEN_CSUM)
3718 f2 |= (NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3719 f1 &= f2;
3720 if (f1 & NETIF_F_GEN_CSUM)
3721 f1 &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
3722
3723 return f1;
3724}
3725
3726static inline netdev_features_t netdev_get_wanted_features(
3727 struct net_device *dev)
3728{
3729 return (dev->features & ~dev->hw_features) | dev->wanted_features;
3730}
3731netdev_features_t netdev_increment_features(netdev_features_t all,
3732 netdev_features_t one, netdev_features_t mask);
3733
3734
3735
3736
3737
3738static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
3739 netdev_features_t mask)
3740{
3741 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
3742}
3743
3744int __netdev_update_features(struct net_device *dev);
3745void netdev_update_features(struct net_device *dev);
3746void netdev_change_features(struct net_device *dev);
3747
3748void netif_stacked_transfer_operstate(const struct net_device *rootdev,
3749 struct net_device *dev);
3750
3751netdev_features_t passthru_features_check(struct sk_buff *skb,
3752 struct net_device *dev,
3753 netdev_features_t features);
3754netdev_features_t netif_skb_features(struct sk_buff *skb);
3755
3756static inline bool net_gso_ok(netdev_features_t features, int gso_type)
3757{
3758 netdev_features_t feature = gso_type << NETIF_F_GSO_SHIFT;
3759
3760
3761 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
3762 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_UFO >> NETIF_F_GSO_SHIFT));
3763 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
3764 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
3765 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
3766 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
3767 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
3768 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
3769 BUILD_BUG_ON(SKB_GSO_IPIP != (NETIF_F_GSO_IPIP >> NETIF_F_GSO_SHIFT));
3770 BUILD_BUG_ON(SKB_GSO_SIT != (NETIF_F_GSO_SIT >> NETIF_F_GSO_SHIFT));
3771 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
3772 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
3773 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
3774
3775 return (features & feature) == feature;
3776}
3777
3778static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
3779{
3780 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
3781 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
3782}
3783
3784static inline bool netif_needs_gso(struct sk_buff *skb,
3785 netdev_features_t features)
3786{
3787 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
3788 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
3789 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
3790}
3791
3792static inline void netif_set_gso_max_size(struct net_device *dev,
3793 unsigned int size)
3794{
3795 dev->gso_max_size = size;
3796}
3797
3798static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
3799 int pulled_hlen, u16 mac_offset,
3800 int mac_len)
3801{
3802 skb->protocol = protocol;
3803 skb->encapsulation = 1;
3804 skb_push(skb, pulled_hlen);
3805 skb_reset_transport_header(skb);
3806 skb->mac_header = mac_offset;
3807 skb->network_header = skb->mac_header + mac_len;
3808 skb->mac_len = mac_len;
3809}
3810
3811static inline bool netif_is_macvlan(struct net_device *dev)
3812{
3813 return dev->priv_flags & IFF_MACVLAN;
3814}
3815
3816static inline bool netif_is_macvlan_port(struct net_device *dev)
3817{
3818 return dev->priv_flags & IFF_MACVLAN_PORT;
3819}
3820
3821static inline bool netif_is_ipvlan(struct net_device *dev)
3822{
3823 return dev->priv_flags & IFF_IPVLAN_SLAVE;
3824}
3825
3826static inline bool netif_is_ipvlan_port(struct net_device *dev)
3827{
3828 return dev->priv_flags & IFF_IPVLAN_MASTER;
3829}
3830
3831static inline bool netif_is_bond_master(struct net_device *dev)
3832{
3833 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
3834}
3835
3836static inline bool netif_is_bond_slave(struct net_device *dev)
3837{
3838 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
3839}
3840
3841static inline bool netif_supports_nofcs(struct net_device *dev)
3842{
3843 return dev->priv_flags & IFF_SUPP_NOFCS;
3844}
3845
3846static inline bool netif_is_l3_master(const struct net_device *dev)
3847{
3848 return dev->priv_flags & IFF_L3MDEV_MASTER;
3849}
3850
3851static inline bool netif_is_l3_slave(const struct net_device *dev)
3852{
3853 return dev->priv_flags & IFF_L3MDEV_SLAVE;
3854}
3855
3856static inline bool netif_is_bridge_master(const struct net_device *dev)
3857{
3858 return dev->priv_flags & IFF_EBRIDGE;
3859}
3860
3861static inline bool netif_is_bridge_port(const struct net_device *dev)
3862{
3863 return dev->priv_flags & IFF_BRIDGE_PORT;
3864}
3865
3866static inline bool netif_is_ovs_master(const struct net_device *dev)
3867{
3868 return dev->priv_flags & IFF_OPENVSWITCH;
3869}
3870
3871
3872static inline void netif_keep_dst(struct net_device *dev)
3873{
3874 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
3875}
3876
3877extern struct pernet_operations __net_initdata loopback_net_ops;
3878
3879
3880
3881
3882
3883static inline const char *netdev_name(const struct net_device *dev)
3884{
3885 if (!dev->name[0] || strchr(dev->name, '%'))
3886 return "(unnamed net_device)";
3887 return dev->name;
3888}
3889
3890static inline const char *netdev_reg_state(const struct net_device *dev)
3891{
3892 switch (dev->reg_state) {
3893 case NETREG_UNINITIALIZED: return " (uninitialized)";
3894 case NETREG_REGISTERED: return "";
3895 case NETREG_UNREGISTERING: return " (unregistering)";
3896 case NETREG_UNREGISTERED: return " (unregistered)";
3897 case NETREG_RELEASED: return " (released)";
3898 case NETREG_DUMMY: return " (dummy)";
3899 }
3900
3901 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
3902 return " (unknown)";
3903}
3904
3905__printf(3, 4)
3906void netdev_printk(const char *level, const struct net_device *dev,
3907 const char *format, ...);
3908__printf(2, 3)
3909void netdev_emerg(const struct net_device *dev, const char *format, ...);
3910__printf(2, 3)
3911void netdev_alert(const struct net_device *dev, const char *format, ...);
3912__printf(2, 3)
3913void netdev_crit(const struct net_device *dev, const char *format, ...);
3914__printf(2, 3)
3915void netdev_err(const struct net_device *dev, const char *format, ...);
3916__printf(2, 3)
3917void netdev_warn(const struct net_device *dev, const char *format, ...);
3918__printf(2, 3)
3919void netdev_notice(const struct net_device *dev, const char *format, ...);
3920__printf(2, 3)
3921void netdev_info(const struct net_device *dev, const char *format, ...);
3922
3923#define MODULE_ALIAS_NETDEV(device) \
3924 MODULE_ALIAS("netdev-" device)
3925
3926#if defined(CONFIG_DYNAMIC_DEBUG)
3927#define netdev_dbg(__dev, format, args...) \
3928do { \
3929 dynamic_netdev_dbg(__dev, format, ##args); \
3930} while (0)
3931#elif defined(DEBUG)
3932#define netdev_dbg(__dev, format, args...) \
3933 netdev_printk(KERN_DEBUG, __dev, format, ##args)
3934#else
3935#define netdev_dbg(__dev, format, args...) \
3936({ \
3937 if (0) \
3938 netdev_printk(KERN_DEBUG, __dev, format, ##args); \
3939})
3940#endif
3941
3942#if defined(VERBOSE_DEBUG)
3943#define netdev_vdbg netdev_dbg
3944#else
3945
3946#define netdev_vdbg(dev, format, args...) \
3947({ \
3948 if (0) \
3949 netdev_printk(KERN_DEBUG, dev, format, ##args); \
3950 0; \
3951})
3952#endif
3953
3954
3955
3956
3957
3958
3959#define netdev_WARN(dev, format, args...) \
3960 WARN(1, "netdevice: %s%s\n" format, netdev_name(dev), \
3961 netdev_reg_state(dev), ##args)
3962
3963
3964
3965#define netif_printk(priv, type, level, dev, fmt, args...) \
3966do { \
3967 if (netif_msg_##type(priv)) \
3968 netdev_printk(level, (dev), fmt, ##args); \
3969} while (0)
3970
3971#define netif_level(level, priv, type, dev, fmt, args...) \
3972do { \
3973 if (netif_msg_##type(priv)) \
3974 netdev_##level(dev, fmt, ##args); \
3975} while (0)
3976
3977#define netif_emerg(priv, type, dev, fmt, args...) \
3978 netif_level(emerg, priv, type, dev, fmt, ##args)
3979#define netif_alert(priv, type, dev, fmt, args...) \
3980 netif_level(alert, priv, type, dev, fmt, ##args)
3981#define netif_crit(priv, type, dev, fmt, args...) \
3982 netif_level(crit, priv, type, dev, fmt, ##args)
3983#define netif_err(priv, type, dev, fmt, args...) \
3984 netif_level(err, priv, type, dev, fmt, ##args)
3985#define netif_warn(priv, type, dev, fmt, args...) \
3986 netif_level(warn, priv, type, dev, fmt, ##args)
3987#define netif_notice(priv, type, dev, fmt, args...) \
3988 netif_level(notice, priv, type, dev, fmt, ##args)
3989#define netif_info(priv, type, dev, fmt, args...) \
3990 netif_level(info, priv, type, dev, fmt, ##args)
3991
3992#if defined(CONFIG_DYNAMIC_DEBUG)
3993#define netif_dbg(priv, type, netdev, format, args...) \
3994do { \
3995 if (netif_msg_##type(priv)) \
3996 dynamic_netdev_dbg(netdev, format, ##args); \
3997} while (0)
3998#elif defined(DEBUG)
3999#define netif_dbg(priv, type, dev, format, args...) \
4000 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
4001#else
4002#define netif_dbg(priv, type, dev, format, args...) \
4003({ \
4004 if (0) \
4005 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4006 0; \
4007})
4008#endif
4009
4010#if defined(VERBOSE_DEBUG)
4011#define netif_vdbg netif_dbg
4012#else
4013#define netif_vdbg(priv, type, dev, format, args...) \
4014({ \
4015 if (0) \
4016 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
4017 0; \
4018})
4019#endif
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048#define PTYPE_HASH_SIZE (16)
4049#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
4050
4051#endif
4052