1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef _LINUX_NETDEVICE_H
26#define _LINUX_NETDEVICE_H
27
28#include <linux/pm_qos.h>
29#include <linux/timer.h>
30#include <linux/bug.h>
31#include <linux/delay.h>
32#include <linux/atomic.h>
33#include <asm/cache.h>
34#include <asm/byteorder.h>
35
36#include <linux/percpu.h>
37#include <linux/rculist.h>
38#include <linux/dmaengine.h>
39#include <linux/workqueue.h>
40#include <linux/dynamic_queue_limits.h>
41
42#include <linux/ethtool.h>
43#include <net/net_namespace.h>
44#include <net/dsa.h>
45#ifdef CONFIG_DCB
46#include <net/dcbnl.h>
47#endif
48#include <net/netprio_cgroup.h>
49
50#include <linux/netdev_features.h>
51#include <linux/neighbour.h>
52#include <uapi/linux/netdevice.h>
53
54struct netpoll_info;
55struct device;
56struct phy_device;
57
58struct wireless_dev;
59
60#define SET_ETHTOOL_OPS(netdev,ops) \
61 ( (netdev)->ethtool_ops = (ops) )
62
63void netdev_set_default_ethtool_ops(struct net_device *dev,
64 const struct ethtool_ops *ops);
65
66
67#define NET_ADDR_PERM 0
68#define NET_ADDR_RANDOM 1
69#define NET_ADDR_STOLEN 2
70#define NET_ADDR_SET 3
71
72
73
74#define NET_RX_SUCCESS 0
75#define NET_RX_DROP 1
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95#define NET_XMIT_SUCCESS 0x00
96#define NET_XMIT_DROP 0x01
97#define NET_XMIT_CN 0x02
98#define NET_XMIT_POLICED 0x03
99#define NET_XMIT_MASK 0x0f
100
101
102
103
104#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
105#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
106
107
108#define NETDEV_TX_MASK 0xf0
109
110enum netdev_tx {
111 __NETDEV_TX_MIN = INT_MIN,
112 NETDEV_TX_OK = 0x00,
113 NETDEV_TX_BUSY = 0x10,
114 NETDEV_TX_LOCKED = 0x20,
115};
116typedef enum netdev_tx netdev_tx_t;
117
118
119
120
121
122static inline bool dev_xmit_complete(int rc)
123{
124
125
126
127
128
129
130 if (likely(rc < NET_XMIT_MASK))
131 return true;
132
133 return false;
134}
135
136
137
138
139
140
141#if defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
142# if defined(CONFIG_MAC80211_MESH)
143# define LL_MAX_HEADER 128
144# else
145# define LL_MAX_HEADER 96
146# endif
147#else
148# define LL_MAX_HEADER 32
149#endif
150
151#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
152 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
153#define MAX_HEADER LL_MAX_HEADER
154#else
155#define MAX_HEADER (LL_MAX_HEADER + 48)
156#endif
157
158
159
160
161
162
163struct net_device_stats {
164 unsigned long rx_packets;
165 unsigned long tx_packets;
166 unsigned long rx_bytes;
167 unsigned long tx_bytes;
168 unsigned long rx_errors;
169 unsigned long tx_errors;
170 unsigned long rx_dropped;
171 unsigned long tx_dropped;
172 unsigned long multicast;
173 unsigned long collisions;
174 unsigned long rx_length_errors;
175 unsigned long rx_over_errors;
176 unsigned long rx_crc_errors;
177 unsigned long rx_frame_errors;
178 unsigned long rx_fifo_errors;
179 unsigned long rx_missed_errors;
180 unsigned long tx_aborted_errors;
181 unsigned long tx_carrier_errors;
182 unsigned long tx_fifo_errors;
183 unsigned long tx_heartbeat_errors;
184 unsigned long tx_window_errors;
185 unsigned long rx_compressed;
186 unsigned long tx_compressed;
187};
188
189
190#include <linux/cache.h>
191#include <linux/skbuff.h>
192
193#ifdef CONFIG_RPS
194#include <linux/static_key.h>
195extern struct static_key rps_needed;
196#endif
197
198struct neighbour;
199struct neigh_parms;
200struct sk_buff;
201
202struct netdev_hw_addr {
203 struct list_head list;
204 unsigned char addr[MAX_ADDR_LEN];
205 unsigned char type;
206#define NETDEV_HW_ADDR_T_LAN 1
207#define NETDEV_HW_ADDR_T_SAN 2
208#define NETDEV_HW_ADDR_T_SLAVE 3
209#define NETDEV_HW_ADDR_T_UNICAST 4
210#define NETDEV_HW_ADDR_T_MULTICAST 5
211 bool global_use;
212 int sync_cnt;
213 int refcount;
214 int synced;
215 struct rcu_head rcu_head;
216};
217
218struct netdev_hw_addr_list {
219 struct list_head list;
220 int count;
221};
222
223#define netdev_hw_addr_list_count(l) ((l)->count)
224#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
225#define netdev_hw_addr_list_for_each(ha, l) \
226 list_for_each_entry(ha, &(l)->list, list)
227
228#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
229#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
230#define netdev_for_each_uc_addr(ha, dev) \
231 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
232
233#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
234#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
235#define netdev_for_each_mc_addr(ha, dev) \
236 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
237
238struct hh_cache {
239 u16 hh_len;
240 u16 __pad;
241 seqlock_t hh_lock;
242
243
244#define HH_DATA_MOD 16
245#define HH_DATA_OFF(__len) \
246 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
247#define HH_DATA_ALIGN(__len) \
248 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
249 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
250};
251
252
253
254
255
256
257
258
259
260#define LL_RESERVED_SPACE(dev) \
261 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
262#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
263 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
264
265struct header_ops {
266 int (*create) (struct sk_buff *skb, struct net_device *dev,
267 unsigned short type, const void *daddr,
268 const void *saddr, unsigned int len);
269 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
270 int (*rebuild)(struct sk_buff *skb);
271 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
272 void (*cache_update)(struct hh_cache *hh,
273 const struct net_device *dev,
274 const unsigned char *haddr);
275};
276
277
278
279
280
281
282enum netdev_state_t {
283 __LINK_STATE_START,
284 __LINK_STATE_PRESENT,
285 __LINK_STATE_NOCARRIER,
286 __LINK_STATE_LINKWATCH_PENDING,
287 __LINK_STATE_DORMANT,
288};
289
290
291
292
293
294
295struct netdev_boot_setup {
296 char name[IFNAMSIZ];
297 struct ifmap map;
298};
299#define NETDEV_BOOT_SETUP_MAX 8
300
301int __init netdev_boot_setup(char *str);
302
303
304
305
306struct napi_struct {
307
308
309
310
311
312
313 struct list_head poll_list;
314
315 unsigned long state;
316 int weight;
317 unsigned int gro_count;
318 int (*poll)(struct napi_struct *, int);
319#ifdef CONFIG_NETPOLL
320 spinlock_t poll_lock;
321 int poll_owner;
322#endif
323 struct net_device *dev;
324 struct sk_buff *gro_list;
325 struct sk_buff *skb;
326 struct list_head dev_list;
327 struct hlist_node napi_hash_node;
328 unsigned int napi_id;
329};
330
331enum {
332 NAPI_STATE_SCHED,
333 NAPI_STATE_DISABLE,
334 NAPI_STATE_NPSVC,
335 NAPI_STATE_HASHED,
336};
337
338enum gro_result {
339 GRO_MERGED,
340 GRO_MERGED_FREE,
341 GRO_HELD,
342 GRO_NORMAL,
343 GRO_DROP,
344};
345typedef enum gro_result gro_result_t;
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388enum rx_handler_result {
389 RX_HANDLER_CONSUMED,
390 RX_HANDLER_ANOTHER,
391 RX_HANDLER_EXACT,
392 RX_HANDLER_PASS,
393};
394typedef enum rx_handler_result rx_handler_result_t;
395typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
396
397void __napi_schedule(struct napi_struct *n);
398
399static inline bool napi_disable_pending(struct napi_struct *n)
400{
401 return test_bit(NAPI_STATE_DISABLE, &n->state);
402}
403
404
405
406
407
408
409
410
411
412
413static inline bool napi_schedule_prep(struct napi_struct *n)
414{
415 return !napi_disable_pending(n) &&
416 !test_and_set_bit(NAPI_STATE_SCHED, &n->state);
417}
418
419
420
421
422
423
424
425
426static inline void napi_schedule(struct napi_struct *n)
427{
428 if (napi_schedule_prep(n))
429 __napi_schedule(n);
430}
431
432
433static inline bool napi_reschedule(struct napi_struct *napi)
434{
435 if (napi_schedule_prep(napi)) {
436 __napi_schedule(napi);
437 return true;
438 }
439 return false;
440}
441
442
443
444
445
446
447
448void __napi_complete(struct napi_struct *n);
449void napi_complete(struct napi_struct *n);
450
451
452
453
454
455
456
457
458struct napi_struct *napi_by_id(unsigned int napi_id);
459
460
461
462
463
464
465
466void napi_hash_add(struct napi_struct *napi);
467
468
469
470
471
472
473
474
475void napi_hash_del(struct napi_struct *napi);
476
477
478
479
480
481
482
483
484static inline void napi_disable(struct napi_struct *n)
485{
486 might_sleep();
487 set_bit(NAPI_STATE_DISABLE, &n->state);
488 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
489 msleep(1);
490 clear_bit(NAPI_STATE_DISABLE, &n->state);
491}
492
493
494
495
496
497
498
499
500static inline void napi_enable(struct napi_struct *n)
501{
502 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
503 smp_mb__before_clear_bit();
504 clear_bit(NAPI_STATE_SCHED, &n->state);
505}
506
507#ifdef CONFIG_SMP
508
509
510
511
512
513
514
515
516static inline void napi_synchronize(const struct napi_struct *n)
517{
518 while (test_bit(NAPI_STATE_SCHED, &n->state))
519 msleep(1);
520}
521#else
522# define napi_synchronize(n) barrier()
523#endif
524
525enum netdev_queue_state_t {
526 __QUEUE_STATE_DRV_XOFF,
527 __QUEUE_STATE_STACK_XOFF,
528 __QUEUE_STATE_FROZEN,
529#define QUEUE_STATE_ANY_XOFF ((1 << __QUEUE_STATE_DRV_XOFF) | \
530 (1 << __QUEUE_STATE_STACK_XOFF))
531#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
532 (1 << __QUEUE_STATE_FROZEN))
533};
534
535
536
537
538
539
540
541
542
543
544struct netdev_queue {
545
546
547
548 struct net_device *dev;
549 struct Qdisc *qdisc;
550 struct Qdisc *qdisc_sleeping;
551#ifdef CONFIG_SYSFS
552 struct kobject kobj;
553#endif
554#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
555 int numa_node;
556#endif
557
558
559
560 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
561 int xmit_lock_owner;
562
563
564
565 unsigned long trans_start;
566
567
568
569
570
571 unsigned long trans_timeout;
572
573 unsigned long state;
574
575#ifdef CONFIG_BQL
576 struct dql dql;
577#endif
578} ____cacheline_aligned_in_smp;
579
580static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
581{
582#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
583 return q->numa_node;
584#else
585 return NUMA_NO_NODE;
586#endif
587}
588
589static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
590{
591#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
592 q->numa_node = node;
593#endif
594}
595
596#ifdef CONFIG_RPS
597
598
599
600
601struct rps_map {
602 unsigned int len;
603 struct rcu_head rcu;
604 u16 cpus[0];
605};
606#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
607
608
609
610
611
612
613struct rps_dev_flow {
614 u16 cpu;
615 u16 filter;
616 unsigned int last_qtail;
617};
618#define RPS_NO_FILTER 0xffff
619
620
621
622
623struct rps_dev_flow_table {
624 unsigned int mask;
625 struct rcu_head rcu;
626 struct rps_dev_flow flows[0];
627};
628#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
629 ((_num) * sizeof(struct rps_dev_flow)))
630
631
632
633
634
635struct rps_sock_flow_table {
636 unsigned int mask;
637 u16 ents[0];
638};
639#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_sock_flow_table) + \
640 ((_num) * sizeof(u16)))
641
642#define RPS_NO_CPU 0xffff
643
644static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
645 u32 hash)
646{
647 if (table && hash) {
648 unsigned int cpu, index = hash & table->mask;
649
650
651 cpu = raw_smp_processor_id();
652
653 if (table->ents[index] != cpu)
654 table->ents[index] = cpu;
655 }
656}
657
658static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
659 u32 hash)
660{
661 if (table && hash)
662 table->ents[hash & table->mask] = RPS_NO_CPU;
663}
664
665extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
666
667#ifdef CONFIG_RFS_ACCEL
668bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
669 u16 filter_id);
670#endif
671#endif
672
673
674struct netdev_rx_queue {
675#ifdef CONFIG_RPS
676 struct rps_map __rcu *rps_map;
677 struct rps_dev_flow_table __rcu *rps_flow_table;
678#endif
679 struct kobject kobj;
680 struct net_device *dev;
681} ____cacheline_aligned_in_smp;
682
683
684
685
686struct rx_queue_attribute {
687 struct attribute attr;
688 ssize_t (*show)(struct netdev_rx_queue *queue,
689 struct rx_queue_attribute *attr, char *buf);
690 ssize_t (*store)(struct netdev_rx_queue *queue,
691 struct rx_queue_attribute *attr, const char *buf, size_t len);
692};
693
694#ifdef CONFIG_XPS
695
696
697
698
699struct xps_map {
700 unsigned int len;
701 unsigned int alloc_len;
702 struct rcu_head rcu;
703 u16 queues[0];
704};
705#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
706#define XPS_MIN_MAP_ALLOC ((L1_CACHE_BYTES - sizeof(struct xps_map)) \
707 / sizeof(u16))
708
709
710
711
712struct xps_dev_maps {
713 struct rcu_head rcu;
714 struct xps_map __rcu *cpu_map[0];
715};
716#define XPS_DEV_MAPS_SIZE (sizeof(struct xps_dev_maps) + \
717 (nr_cpu_ids * sizeof(struct xps_map *)))
718#endif
719
720#define TC_MAX_QUEUE 16
721#define TC_BITMASK 15
722
723struct netdev_tc_txq {
724 u16 count;
725 u16 offset;
726};
727
728#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
729
730
731
732
733struct netdev_fcoe_hbainfo {
734 char manufacturer[64];
735 char serial_number[64];
736 char hardware_version[64];
737 char driver_version[64];
738 char optionrom_version[64];
739 char firmware_version[64];
740 char model[256];
741 char model_description[256];
742};
743#endif
744
745#define MAX_PHYS_PORT_ID_LEN 32
746
747
748
749
750struct netdev_phys_port_id {
751 unsigned char id[MAX_PHYS_PORT_ID_LEN];
752 unsigned char id_len;
753};
754
755typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
756 struct sk_buff *skb);
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002struct net_device_ops {
1003 int (*ndo_init)(struct net_device *dev);
1004 void (*ndo_uninit)(struct net_device *dev);
1005 int (*ndo_open)(struct net_device *dev);
1006 int (*ndo_stop)(struct net_device *dev);
1007 netdev_tx_t (*ndo_start_xmit) (struct sk_buff *skb,
1008 struct net_device *dev);
1009 u16 (*ndo_select_queue)(struct net_device *dev,
1010 struct sk_buff *skb,
1011 void *accel_priv,
1012 select_queue_fallback_t fallback);
1013 void (*ndo_change_rx_flags)(struct net_device *dev,
1014 int flags);
1015 void (*ndo_set_rx_mode)(struct net_device *dev);
1016 int (*ndo_set_mac_address)(struct net_device *dev,
1017 void *addr);
1018 int (*ndo_validate_addr)(struct net_device *dev);
1019 int (*ndo_do_ioctl)(struct net_device *dev,
1020 struct ifreq *ifr, int cmd);
1021 int (*ndo_set_config)(struct net_device *dev,
1022 struct ifmap *map);
1023 int (*ndo_change_mtu)(struct net_device *dev,
1024 int new_mtu);
1025 int (*ndo_neigh_setup)(struct net_device *dev,
1026 struct neigh_parms *);
1027 void (*ndo_tx_timeout) (struct net_device *dev);
1028
1029 struct rtnl_link_stats64* (*ndo_get_stats64)(struct net_device *dev,
1030 struct rtnl_link_stats64 *storage);
1031 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1032
1033 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1034 __be16 proto, u16 vid);
1035 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1036 __be16 proto, u16 vid);
1037#ifdef CONFIG_NET_POLL_CONTROLLER
1038 void (*ndo_poll_controller)(struct net_device *dev);
1039 int (*ndo_netpoll_setup)(struct net_device *dev,
1040 struct netpoll_info *info,
1041 gfp_t gfp);
1042 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1043#endif
1044#ifdef CONFIG_NET_RX_BUSY_POLL
1045 int (*ndo_busy_poll)(struct napi_struct *dev);
1046#endif
1047 int (*ndo_set_vf_mac)(struct net_device *dev,
1048 int queue, u8 *mac);
1049 int (*ndo_set_vf_vlan)(struct net_device *dev,
1050 int queue, u16 vlan, u8 qos);
1051 int (*ndo_set_vf_tx_rate)(struct net_device *dev,
1052 int vf, int rate);
1053 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1054 int vf, bool setting);
1055 int (*ndo_get_vf_config)(struct net_device *dev,
1056 int vf,
1057 struct ifla_vf_info *ivf);
1058 int (*ndo_set_vf_link_state)(struct net_device *dev,
1059 int vf, int link_state);
1060 int (*ndo_set_vf_port)(struct net_device *dev,
1061 int vf,
1062 struct nlattr *port[]);
1063 int (*ndo_get_vf_port)(struct net_device *dev,
1064 int vf, struct sk_buff *skb);
1065 int (*ndo_setup_tc)(struct net_device *dev, u8 tc);
1066#if IS_ENABLED(CONFIG_FCOE)
1067 int (*ndo_fcoe_enable)(struct net_device *dev);
1068 int (*ndo_fcoe_disable)(struct net_device *dev);
1069 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1070 u16 xid,
1071 struct scatterlist *sgl,
1072 unsigned int sgc);
1073 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1074 u16 xid);
1075 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1076 u16 xid,
1077 struct scatterlist *sgl,
1078 unsigned int sgc);
1079 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1080 struct netdev_fcoe_hbainfo *hbainfo);
1081#endif
1082
1083#if IS_ENABLED(CONFIG_LIBFCOE)
1084#define NETDEV_FCOE_WWNN 0
1085#define NETDEV_FCOE_WWPN 1
1086 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1087 u64 *wwn, int type);
1088#endif
1089
1090#ifdef CONFIG_RFS_ACCEL
1091 int (*ndo_rx_flow_steer)(struct net_device *dev,
1092 const struct sk_buff *skb,
1093 u16 rxq_index,
1094 u32 flow_id);
1095#endif
1096 int (*ndo_add_slave)(struct net_device *dev,
1097 struct net_device *slave_dev);
1098 int (*ndo_del_slave)(struct net_device *dev,
1099 struct net_device *slave_dev);
1100 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1101 netdev_features_t features);
1102 int (*ndo_set_features)(struct net_device *dev,
1103 netdev_features_t features);
1104 int (*ndo_neigh_construct)(struct neighbour *n);
1105 void (*ndo_neigh_destroy)(struct neighbour *n);
1106
1107 int (*ndo_fdb_add)(struct ndmsg *ndm,
1108 struct nlattr *tb[],
1109 struct net_device *dev,
1110 const unsigned char *addr,
1111 u16 flags);
1112 int (*ndo_fdb_del)(struct ndmsg *ndm,
1113 struct nlattr *tb[],
1114 struct net_device *dev,
1115 const unsigned char *addr);
1116 int (*ndo_fdb_dump)(struct sk_buff *skb,
1117 struct netlink_callback *cb,
1118 struct net_device *dev,
1119 int idx);
1120
1121 int (*ndo_bridge_setlink)(struct net_device *dev,
1122 struct nlmsghdr *nlh);
1123 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1124 u32 pid, u32 seq,
1125 struct net_device *dev,
1126 u32 filter_mask);
1127 int (*ndo_bridge_dellink)(struct net_device *dev,
1128 struct nlmsghdr *nlh);
1129 int (*ndo_change_carrier)(struct net_device *dev,
1130 bool new_carrier);
1131 int (*ndo_get_phys_port_id)(struct net_device *dev,
1132 struct netdev_phys_port_id *ppid);
1133 void (*ndo_add_vxlan_port)(struct net_device *dev,
1134 sa_family_t sa_family,
1135 __be16 port);
1136 void (*ndo_del_vxlan_port)(struct net_device *dev,
1137 sa_family_t sa_family,
1138 __be16 port);
1139
1140 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1141 struct net_device *dev);
1142 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1143 void *priv);
1144
1145 netdev_tx_t (*ndo_dfwd_start_xmit) (struct sk_buff *skb,
1146 struct net_device *dev,
1147 void *priv);
1148};
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160struct net_device {
1161
1162
1163
1164
1165
1166
1167 char name[IFNAMSIZ];
1168
1169
1170 struct hlist_node name_hlist;
1171
1172
1173 char *ifalias;
1174
1175
1176
1177
1178
1179 unsigned long mem_end;
1180 unsigned long mem_start;
1181 unsigned long base_addr;
1182 int irq;
1183
1184
1185
1186
1187
1188
1189 unsigned long state;
1190
1191 struct list_head dev_list;
1192 struct list_head napi_list;
1193 struct list_head unreg_list;
1194 struct list_head close_list;
1195
1196
1197 struct {
1198 struct list_head upper;
1199 struct list_head lower;
1200 } adj_list;
1201
1202
1203 struct {
1204 struct list_head upper;
1205 struct list_head lower;
1206 } all_adj_list;
1207
1208
1209
1210 netdev_features_t features;
1211
1212 netdev_features_t hw_features;
1213
1214 netdev_features_t wanted_features;
1215
1216 netdev_features_t vlan_features;
1217
1218
1219
1220
1221
1222 netdev_features_t hw_enc_features;
1223
1224 netdev_features_t mpls_features;
1225
1226
1227 int ifindex;
1228 int iflink;
1229
1230 struct net_device_stats stats;
1231 atomic_long_t rx_dropped;
1232
1233
1234
1235#ifdef CONFIG_WIRELESS_EXT
1236
1237
1238 const struct iw_handler_def * wireless_handlers;
1239
1240 struct iw_public_data * wireless_data;
1241#endif
1242
1243 const struct net_device_ops *netdev_ops;
1244 const struct ethtool_ops *ethtool_ops;
1245 const struct forwarding_accel_ops *fwd_ops;
1246
1247
1248 const struct header_ops *header_ops;
1249
1250 unsigned int flags;
1251 unsigned int priv_flags;
1252
1253 unsigned short gflags;
1254 unsigned short padded;
1255
1256 unsigned char operstate;
1257 unsigned char link_mode;
1258
1259 unsigned char if_port;
1260 unsigned char dma;
1261
1262 unsigned int mtu;
1263 unsigned short type;
1264 unsigned short hard_header_len;
1265
1266
1267
1268
1269
1270 unsigned short needed_headroom;
1271 unsigned short needed_tailroom;
1272
1273
1274 unsigned char perm_addr[MAX_ADDR_LEN];
1275 unsigned char addr_assign_type;
1276 unsigned char addr_len;
1277 unsigned short neigh_priv_len;
1278 unsigned short dev_id;
1279
1280
1281
1282 spinlock_t addr_list_lock;
1283 struct netdev_hw_addr_list uc;
1284 struct netdev_hw_addr_list mc;
1285 struct netdev_hw_addr_list dev_addrs;
1286
1287
1288#ifdef CONFIG_SYSFS
1289 struct kset *queues_kset;
1290#endif
1291
1292 bool uc_promisc;
1293 unsigned int promiscuity;
1294 unsigned int allmulti;
1295
1296
1297
1298
1299#if IS_ENABLED(CONFIG_VLAN_8021Q)
1300 struct vlan_info __rcu *vlan_info;
1301#endif
1302#if IS_ENABLED(CONFIG_NET_DSA)
1303 struct dsa_switch_tree *dsa_ptr;
1304#endif
1305#if IS_ENABLED(CONFIG_TIPC)
1306 struct tipc_bearer __rcu *tipc_ptr;
1307#endif
1308 void *atalk_ptr;
1309 struct in_device __rcu *ip_ptr;
1310 struct dn_dev __rcu *dn_ptr;
1311 struct inet6_dev __rcu *ip6_ptr;
1312 void *ax25_ptr;
1313 struct wireless_dev *ieee80211_ptr;
1314
1315
1316
1317
1318
1319 unsigned long last_rx;
1320
1321
1322
1323
1324
1325
1326
1327
1328 unsigned char *dev_addr;
1329
1330
1331
1332
1333#ifdef CONFIG_SYSFS
1334 struct netdev_rx_queue *_rx;
1335
1336
1337 unsigned int num_rx_queues;
1338
1339
1340 unsigned int real_num_rx_queues;
1341
1342#endif
1343
1344 rx_handler_func_t __rcu *rx_handler;
1345 void __rcu *rx_handler_data;
1346
1347 struct netdev_queue __rcu *ingress_queue;
1348 unsigned char broadcast[MAX_ADDR_LEN];
1349
1350
1351
1352
1353
1354 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
1355
1356
1357 unsigned int num_tx_queues;
1358
1359
1360 unsigned int real_num_tx_queues;
1361
1362
1363 struct Qdisc *qdisc;
1364
1365 unsigned long tx_queue_len;
1366 spinlock_t tx_global_lock;
1367
1368#ifdef CONFIG_XPS
1369 struct xps_dev_maps __rcu *xps_maps;
1370#endif
1371#ifdef CONFIG_RFS_ACCEL
1372
1373
1374
1375 struct cpu_rmap *rx_cpu_rmap;
1376#endif
1377
1378
1379
1380
1381
1382
1383
1384 unsigned long trans_start;
1385
1386 int watchdog_timeo;
1387 struct timer_list watchdog_timer;
1388
1389
1390 int __percpu *pcpu_refcnt;
1391
1392
1393 struct list_head todo_list;
1394
1395 struct hlist_node index_hlist;
1396
1397 struct list_head link_watch_list;
1398
1399
1400 enum { NETREG_UNINITIALIZED=0,
1401 NETREG_REGISTERED,
1402 NETREG_UNREGISTERING,
1403 NETREG_UNREGISTERED,
1404 NETREG_RELEASED,
1405 NETREG_DUMMY,
1406 } reg_state:8;
1407
1408 bool dismantle;
1409
1410 enum {
1411 RTNL_LINK_INITIALIZED,
1412 RTNL_LINK_INITIALIZING,
1413 } rtnl_link_state:16;
1414
1415
1416 void (*destructor)(struct net_device *dev);
1417
1418#ifdef CONFIG_NETPOLL
1419 struct netpoll_info __rcu *npinfo;
1420#endif
1421
1422#ifdef CONFIG_NET_NS
1423
1424 struct net *nd_net;
1425#endif
1426
1427
1428 union {
1429 void *ml_priv;
1430 struct pcpu_lstats __percpu *lstats;
1431 struct pcpu_sw_netstats __percpu *tstats;
1432 struct pcpu_dstats __percpu *dstats;
1433 struct pcpu_vstats __percpu *vstats;
1434 };
1435
1436 struct garp_port __rcu *garp_port;
1437
1438 struct mrp_port __rcu *mrp_port;
1439
1440
1441 struct device dev;
1442
1443 const struct attribute_group *sysfs_groups[4];
1444
1445 const struct attribute_group *sysfs_rx_queue_group;
1446
1447
1448 const struct rtnl_link_ops *rtnl_link_ops;
1449
1450
1451#define GSO_MAX_SIZE 65536
1452 unsigned int gso_max_size;
1453#define GSO_MAX_SEGS 65535
1454 u16 gso_max_segs;
1455
1456#ifdef CONFIG_DCB
1457
1458 const struct dcbnl_rtnl_ops *dcbnl_ops;
1459#endif
1460 u8 num_tc;
1461 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
1462 u8 prio_tc_map[TC_BITMASK + 1];
1463
1464#if IS_ENABLED(CONFIG_FCOE)
1465
1466 unsigned int fcoe_ddp_xid;
1467#endif
1468#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
1469 struct netprio_map __rcu *priomap;
1470#endif
1471
1472 struct phy_device *phydev;
1473
1474 struct lock_class_key *qdisc_tx_busylock;
1475
1476
1477 int group;
1478
1479 struct pm_qos_request pm_qos_req;
1480};
1481#define to_net_dev(d) container_of(d, struct net_device, dev)
1482
1483#define NETDEV_ALIGN 32
1484
1485static inline
1486int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
1487{
1488 return dev->prio_tc_map[prio & TC_BITMASK];
1489}
1490
1491static inline
1492int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
1493{
1494 if (tc >= dev->num_tc)
1495 return -EINVAL;
1496
1497 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
1498 return 0;
1499}
1500
1501static inline
1502void netdev_reset_tc(struct net_device *dev)
1503{
1504 dev->num_tc = 0;
1505 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
1506 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
1507}
1508
1509static inline
1510int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
1511{
1512 if (tc >= dev->num_tc)
1513 return -EINVAL;
1514
1515 dev->tc_to_txq[tc].count = count;
1516 dev->tc_to_txq[tc].offset = offset;
1517 return 0;
1518}
1519
1520static inline
1521int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
1522{
1523 if (num_tc > TC_MAX_QUEUE)
1524 return -EINVAL;
1525
1526 dev->num_tc = num_tc;
1527 return 0;
1528}
1529
1530static inline
1531int netdev_get_num_tc(struct net_device *dev)
1532{
1533 return dev->num_tc;
1534}
1535
1536static inline
1537struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
1538 unsigned int index)
1539{
1540 return &dev->_tx[index];
1541}
1542
1543static inline void netdev_for_each_tx_queue(struct net_device *dev,
1544 void (*f)(struct net_device *,
1545 struct netdev_queue *,
1546 void *),
1547 void *arg)
1548{
1549 unsigned int i;
1550
1551 for (i = 0; i < dev->num_tx_queues; i++)
1552 f(dev, &dev->_tx[i], arg);
1553}
1554
1555struct netdev_queue *netdev_pick_tx(struct net_device *dev,
1556 struct sk_buff *skb,
1557 void *accel_priv);
1558
1559
1560
1561
1562static inline
1563struct net *dev_net(const struct net_device *dev)
1564{
1565 return read_pnet(&dev->nd_net);
1566}
1567
1568static inline
1569void dev_net_set(struct net_device *dev, struct net *net)
1570{
1571#ifdef CONFIG_NET_NS
1572 release_net(dev->nd_net);
1573 dev->nd_net = hold_net(net);
1574#endif
1575}
1576
1577static inline bool netdev_uses_dsa_tags(struct net_device *dev)
1578{
1579#ifdef CONFIG_NET_DSA_TAG_DSA
1580 if (dev->dsa_ptr != NULL)
1581 return dsa_uses_dsa_tags(dev->dsa_ptr);
1582#endif
1583
1584 return 0;
1585}
1586
1587static inline bool netdev_uses_trailer_tags(struct net_device *dev)
1588{
1589#ifdef CONFIG_NET_DSA_TAG_TRAILER
1590 if (dev->dsa_ptr != NULL)
1591 return dsa_uses_trailer_tags(dev->dsa_ptr);
1592#endif
1593
1594 return 0;
1595}
1596
1597
1598
1599
1600
1601
1602
1603static inline void *netdev_priv(const struct net_device *dev)
1604{
1605 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
1606}
1607
1608
1609
1610
1611#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
1612
1613
1614
1615
1616
1617#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
1618
1619
1620
1621
1622#define NAPI_POLL_WEIGHT 64
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
1635 int (*poll)(struct napi_struct *, int), int weight);
1636
1637
1638
1639
1640
1641
1642
1643void netif_napi_del(struct napi_struct *napi);
1644
1645struct napi_gro_cb {
1646
1647 void *frag0;
1648
1649
1650 unsigned int frag0_len;
1651
1652
1653 int data_offset;
1654
1655
1656 u16 flush;
1657
1658
1659 u16 flush_id;
1660
1661
1662 u16 count;
1663
1664
1665 u8 same_flow;
1666
1667
1668 u8 free;
1669#define NAPI_GRO_FREE 1
1670#define NAPI_GRO_FREE_STOLEN_HEAD 2
1671
1672
1673 unsigned long age;
1674
1675
1676 u16 proto;
1677
1678
1679 u16 udp_mark;
1680
1681
1682 __wsum csum;
1683
1684
1685 struct sk_buff *last;
1686};
1687
1688#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
1689
1690struct packet_type {
1691 __be16 type;
1692 struct net_device *dev;
1693 int (*func) (struct sk_buff *,
1694 struct net_device *,
1695 struct packet_type *,
1696 struct net_device *);
1697 bool (*id_match)(struct packet_type *ptype,
1698 struct sock *sk);
1699 void *af_packet_priv;
1700 struct list_head list;
1701};
1702
1703struct offload_callbacks {
1704 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
1705 netdev_features_t features);
1706 int (*gso_send_check)(struct sk_buff *skb);
1707 struct sk_buff **(*gro_receive)(struct sk_buff **head,
1708 struct sk_buff *skb);
1709 int (*gro_complete)(struct sk_buff *skb, int nhoff);
1710};
1711
1712struct packet_offload {
1713 __be16 type;
1714 struct offload_callbacks callbacks;
1715 struct list_head list;
1716};
1717
1718struct udp_offload {
1719 __be16 port;
1720 struct offload_callbacks callbacks;
1721};
1722
1723
1724struct pcpu_sw_netstats {
1725 u64 rx_packets;
1726 u64 rx_bytes;
1727 u64 tx_packets;
1728 u64 tx_bytes;
1729 struct u64_stats_sync syncp;
1730};
1731
1732#include <linux/notifier.h>
1733
1734
1735
1736
1737
1738#define NETDEV_UP 0x0001
1739#define NETDEV_DOWN 0x0002
1740#define NETDEV_REBOOT 0x0003
1741
1742
1743
1744#define NETDEV_CHANGE 0x0004
1745#define NETDEV_REGISTER 0x0005
1746#define NETDEV_UNREGISTER 0x0006
1747#define NETDEV_CHANGEMTU 0x0007
1748#define NETDEV_CHANGEADDR 0x0008
1749#define NETDEV_GOING_DOWN 0x0009
1750#define NETDEV_CHANGENAME 0x000A
1751#define NETDEV_FEAT_CHANGE 0x000B
1752#define NETDEV_BONDING_FAILOVER 0x000C
1753#define NETDEV_PRE_UP 0x000D
1754#define NETDEV_PRE_TYPE_CHANGE 0x000E
1755#define NETDEV_POST_TYPE_CHANGE 0x000F
1756#define NETDEV_POST_INIT 0x0010
1757#define NETDEV_UNREGISTER_FINAL 0x0011
1758#define NETDEV_RELEASE 0x0012
1759#define NETDEV_NOTIFY_PEERS 0x0013
1760#define NETDEV_JOIN 0x0014
1761#define NETDEV_CHANGEUPPER 0x0015
1762#define NETDEV_RESEND_IGMP 0x0016
1763#define NETDEV_PRECHANGEMTU 0x0017
1764
1765int register_netdevice_notifier(struct notifier_block *nb);
1766int unregister_netdevice_notifier(struct notifier_block *nb);
1767
1768struct netdev_notifier_info {
1769 struct net_device *dev;
1770};
1771
1772struct netdev_notifier_change_info {
1773 struct netdev_notifier_info info;
1774 unsigned int flags_changed;
1775};
1776
1777static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
1778 struct net_device *dev)
1779{
1780 info->dev = dev;
1781}
1782
1783static inline struct net_device *
1784netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
1785{
1786 return info->dev;
1787}
1788
1789int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
1790
1791
1792extern rwlock_t dev_base_lock;
1793
1794#define for_each_netdev(net, d) \
1795 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
1796#define for_each_netdev_reverse(net, d) \
1797 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
1798#define for_each_netdev_rcu(net, d) \
1799 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
1800#define for_each_netdev_safe(net, d, n) \
1801 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
1802#define for_each_netdev_continue(net, d) \
1803 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
1804#define for_each_netdev_continue_rcu(net, d) \
1805 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
1806#define for_each_netdev_in_bond_rcu(bond, slave) \
1807 for_each_netdev_rcu(&init_net, slave) \
1808 if (netdev_master_upper_dev_get_rcu(slave) == bond)
1809#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
1810
1811static inline struct net_device *next_net_device(struct net_device *dev)
1812{
1813 struct list_head *lh;
1814 struct net *net;
1815
1816 net = dev_net(dev);
1817 lh = dev->dev_list.next;
1818 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
1819}
1820
1821static inline struct net_device *next_net_device_rcu(struct net_device *dev)
1822{
1823 struct list_head *lh;
1824 struct net *net;
1825
1826 net = dev_net(dev);
1827 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
1828 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
1829}
1830
1831static inline struct net_device *first_net_device(struct net *net)
1832{
1833 return list_empty(&net->dev_base_head) ? NULL :
1834 net_device_entry(net->dev_base_head.next);
1835}
1836
1837static inline struct net_device *first_net_device_rcu(struct net *net)
1838{
1839 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
1840
1841 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
1842}
1843
1844int netdev_boot_setup_check(struct net_device *dev);
1845unsigned long netdev_boot_base(const char *prefix, int unit);
1846struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1847 const char *hwaddr);
1848struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
1849struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
1850void dev_add_pack(struct packet_type *pt);
1851void dev_remove_pack(struct packet_type *pt);
1852void __dev_remove_pack(struct packet_type *pt);
1853void dev_add_offload(struct packet_offload *po);
1854void dev_remove_offload(struct packet_offload *po);
1855
1856struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short flags,
1857 unsigned short mask);
1858struct net_device *dev_get_by_name(struct net *net, const char *name);
1859struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
1860struct net_device *__dev_get_by_name(struct net *net, const char *name);
1861int dev_alloc_name(struct net_device *dev, const char *name);
1862int dev_open(struct net_device *dev);
1863int dev_close(struct net_device *dev);
1864void dev_disable_lro(struct net_device *dev);
1865int dev_loopback_xmit(struct sk_buff *newskb);
1866int dev_queue_xmit(struct sk_buff *skb);
1867int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv);
1868int register_netdevice(struct net_device *dev);
1869void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
1870void unregister_netdevice_many(struct list_head *head);
1871static inline void unregister_netdevice(struct net_device *dev)
1872{
1873 unregister_netdevice_queue(dev, NULL);
1874}
1875
1876int netdev_refcnt_read(const struct net_device *dev);
1877void free_netdev(struct net_device *dev);
1878void netdev_freemem(struct net_device *dev);
1879void synchronize_net(void);
1880int init_dummy_netdev(struct net_device *dev);
1881
1882struct net_device *dev_get_by_index(struct net *net, int ifindex);
1883struct net_device *__dev_get_by_index(struct net *net, int ifindex);
1884struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
1885int netdev_get_name(struct net *net, char *name, int ifindex);
1886int dev_restart(struct net_device *dev);
1887#ifdef CONFIG_NETPOLL_TRAP
1888int netpoll_trap(void);
1889#endif
1890int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1891
1892static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
1893{
1894 return NAPI_GRO_CB(skb)->data_offset;
1895}
1896
1897static inline unsigned int skb_gro_len(const struct sk_buff *skb)
1898{
1899 return skb->len - NAPI_GRO_CB(skb)->data_offset;
1900}
1901
1902static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
1903{
1904 NAPI_GRO_CB(skb)->data_offset += len;
1905}
1906
1907static inline void *skb_gro_header_fast(struct sk_buff *skb,
1908 unsigned int offset)
1909{
1910 return NAPI_GRO_CB(skb)->frag0 + offset;
1911}
1912
1913static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
1914{
1915 return NAPI_GRO_CB(skb)->frag0_len < hlen;
1916}
1917
1918static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
1919 unsigned int offset)
1920{
1921 if (!pskb_may_pull(skb, hlen))
1922 return NULL;
1923
1924 NAPI_GRO_CB(skb)->frag0 = NULL;
1925 NAPI_GRO_CB(skb)->frag0_len = 0;
1926 return skb->data + offset;
1927}
1928
1929static inline void *skb_gro_mac_header(struct sk_buff *skb)
1930{
1931 return NAPI_GRO_CB(skb)->frag0 ?: skb_mac_header(skb);
1932}
1933
1934static inline void *skb_gro_network_header(struct sk_buff *skb)
1935{
1936 return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
1937 skb_network_offset(skb);
1938}
1939
1940static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
1941 const void *start, unsigned int len)
1942{
1943 if (skb->ip_summed == CHECKSUM_COMPLETE)
1944 NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
1945 csum_partial(start, len, 0));
1946}
1947
1948static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
1949 unsigned short type,
1950 const void *daddr, const void *saddr,
1951 unsigned int len)
1952{
1953 if (!dev->header_ops || !dev->header_ops->create)
1954 return 0;
1955
1956 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
1957}
1958
1959static inline int dev_parse_header(const struct sk_buff *skb,
1960 unsigned char *haddr)
1961{
1962 const struct net_device *dev = skb->dev;
1963
1964 if (!dev->header_ops || !dev->header_ops->parse)
1965 return 0;
1966 return dev->header_ops->parse(skb, haddr);
1967}
1968
1969static inline int dev_rebuild_header(struct sk_buff *skb)
1970{
1971 const struct net_device *dev = skb->dev;
1972
1973 if (!dev->header_ops || !dev->header_ops->rebuild)
1974 return 0;
1975 return dev->header_ops->rebuild(skb);
1976}
1977
1978typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr, int len);
1979int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
1980static inline int unregister_gifconf(unsigned int family)
1981{
1982 return register_gifconf(family, NULL);
1983}
1984
1985#ifdef CONFIG_NET_FLOW_LIMIT
1986#define FLOW_LIMIT_HISTORY (1 << 7)
1987struct sd_flow_limit {
1988 u64 count;
1989 unsigned int num_buckets;
1990 unsigned int history_head;
1991 u16 history[FLOW_LIMIT_HISTORY];
1992 u8 buckets[];
1993};
1994
1995extern int netdev_flow_limit_table_len;
1996#endif
1997
1998
1999
2000
2001struct softnet_data {
2002 struct Qdisc *output_queue;
2003 struct Qdisc **output_queue_tailp;
2004 struct list_head poll_list;
2005 struct sk_buff *completion_queue;
2006 struct sk_buff_head process_queue;
2007
2008
2009 unsigned int processed;
2010 unsigned int time_squeeze;
2011 unsigned int cpu_collision;
2012 unsigned int received_rps;
2013
2014#ifdef CONFIG_RPS
2015 struct softnet_data *rps_ipi_list;
2016
2017
2018 struct call_single_data csd ____cacheline_aligned_in_smp;
2019 struct softnet_data *rps_ipi_next;
2020 unsigned int cpu;
2021 unsigned int input_queue_head;
2022 unsigned int input_queue_tail;
2023#endif
2024 unsigned int dropped;
2025 struct sk_buff_head input_pkt_queue;
2026 struct napi_struct backlog;
2027
2028#ifdef CONFIG_NET_FLOW_LIMIT
2029 struct sd_flow_limit __rcu *flow_limit;
2030#endif
2031};
2032
2033static inline void input_queue_head_incr(struct softnet_data *sd)
2034{
2035#ifdef CONFIG_RPS
2036 sd->input_queue_head++;
2037#endif
2038}
2039
2040static inline void input_queue_tail_incr_save(struct softnet_data *sd,
2041 unsigned int *qtail)
2042{
2043#ifdef CONFIG_RPS
2044 *qtail = ++sd->input_queue_tail;
2045#endif
2046}
2047
2048DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
2049
2050void __netif_schedule(struct Qdisc *q);
2051
2052static inline void netif_schedule_queue(struct netdev_queue *txq)
2053{
2054 if (!(txq->state & QUEUE_STATE_ANY_XOFF))
2055 __netif_schedule(txq->qdisc);
2056}
2057
2058static inline void netif_tx_schedule_all(struct net_device *dev)
2059{
2060 unsigned int i;
2061
2062 for (i = 0; i < dev->num_tx_queues; i++)
2063 netif_schedule_queue(netdev_get_tx_queue(dev, i));
2064}
2065
2066static inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
2067{
2068 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2069}
2070
2071
2072
2073
2074
2075
2076
2077static inline void netif_start_queue(struct net_device *dev)
2078{
2079 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
2080}
2081
2082static inline void netif_tx_start_all_queues(struct net_device *dev)
2083{
2084 unsigned int i;
2085
2086 for (i = 0; i < dev->num_tx_queues; i++) {
2087 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2088 netif_tx_start_queue(txq);
2089 }
2090}
2091
2092static inline void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2093{
2094#ifdef CONFIG_NETPOLL_TRAP
2095 if (netpoll_trap()) {
2096 netif_tx_start_queue(dev_queue);
2097 return;
2098 }
2099#endif
2100 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state))
2101 __netif_schedule(dev_queue->qdisc);
2102}
2103
2104
2105
2106
2107
2108
2109
2110
2111static inline void netif_wake_queue(struct net_device *dev)
2112{
2113 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
2114}
2115
2116static inline void netif_tx_wake_all_queues(struct net_device *dev)
2117{
2118 unsigned int i;
2119
2120 for (i = 0; i < dev->num_tx_queues; i++) {
2121 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2122 netif_tx_wake_queue(txq);
2123 }
2124}
2125
2126static inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
2127{
2128 if (WARN_ON(!dev_queue)) {
2129 pr_info("netif_stop_queue() cannot be called before register_netdev()\n");
2130 return;
2131 }
2132 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2133}
2134
2135
2136
2137
2138
2139
2140
2141
2142static inline void netif_stop_queue(struct net_device *dev)
2143{
2144 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
2145}
2146
2147static inline void netif_tx_stop_all_queues(struct net_device *dev)
2148{
2149 unsigned int i;
2150
2151 for (i = 0; i < dev->num_tx_queues; i++) {
2152 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2153 netif_tx_stop_queue(txq);
2154 }
2155}
2156
2157static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
2158{
2159 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
2160}
2161
2162
2163
2164
2165
2166
2167
2168static inline bool netif_queue_stopped(const struct net_device *dev)
2169{
2170 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
2171}
2172
2173static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
2174{
2175 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
2176}
2177
2178static inline bool netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
2179{
2180 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
2181}
2182
2183static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
2184 unsigned int bytes)
2185{
2186#ifdef CONFIG_BQL
2187 dql_queued(&dev_queue->dql, bytes);
2188
2189 if (likely(dql_avail(&dev_queue->dql) >= 0))
2190 return;
2191
2192 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2193
2194
2195
2196
2197
2198
2199 smp_mb();
2200
2201
2202 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
2203 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
2204#endif
2205}
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
2217{
2218 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
2219}
2220
2221static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
2222 unsigned int pkts, unsigned int bytes)
2223{
2224#ifdef CONFIG_BQL
2225 if (unlikely(!bytes))
2226 return;
2227
2228 dql_completed(&dev_queue->dql, bytes);
2229
2230
2231
2232
2233
2234
2235 smp_mb();
2236
2237 if (dql_avail(&dev_queue->dql) < 0)
2238 return;
2239
2240 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
2241 netif_schedule_queue(dev_queue);
2242#endif
2243}
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255static inline void netdev_completed_queue(struct net_device *dev,
2256 unsigned int pkts, unsigned int bytes)
2257{
2258 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
2259}
2260
2261static inline void netdev_tx_reset_queue(struct netdev_queue *q)
2262{
2263#ifdef CONFIG_BQL
2264 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
2265 dql_reset(&q->dql);
2266#endif
2267}
2268
2269
2270
2271
2272
2273
2274
2275
2276static inline void netdev_reset_queue(struct net_device *dev_queue)
2277{
2278 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
2279}
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
2290{
2291 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2292 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2293 dev->name, queue_index,
2294 dev->real_num_tx_queues);
2295 return 0;
2296 }
2297
2298 return queue_index;
2299}
2300
2301
2302
2303
2304
2305
2306
2307static inline bool netif_running(const struct net_device *dev)
2308{
2309 return test_bit(__LINK_STATE_START, &dev->state);
2310}
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
2327{
2328 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2329
2330 netif_tx_start_queue(txq);
2331}
2332
2333
2334
2335
2336
2337
2338
2339
2340static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
2341{
2342 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2343#ifdef CONFIG_NETPOLL_TRAP
2344 if (netpoll_trap())
2345 return;
2346#endif
2347 netif_tx_stop_queue(txq);
2348}
2349
2350
2351
2352
2353
2354
2355
2356
2357static inline bool __netif_subqueue_stopped(const struct net_device *dev,
2358 u16 queue_index)
2359{
2360 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2361
2362 return netif_tx_queue_stopped(txq);
2363}
2364
2365static inline bool netif_subqueue_stopped(const struct net_device *dev,
2366 struct sk_buff *skb)
2367{
2368 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
2369}
2370
2371
2372
2373
2374
2375
2376
2377
2378static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2379{
2380 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2381#ifdef CONFIG_NETPOLL_TRAP
2382 if (netpoll_trap())
2383 return;
2384#endif
2385 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state))
2386 __netif_schedule(txq->qdisc);
2387}
2388
2389#ifdef CONFIG_XPS
2390int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2391 u16 index);
2392#else
2393static inline int netif_set_xps_queue(struct net_device *dev,
2394 const struct cpumask *mask,
2395 u16 index)
2396{
2397 return 0;
2398}
2399#endif
2400
2401
2402
2403
2404
2405static inline u16 skb_tx_hash(const struct net_device *dev,
2406 const struct sk_buff *skb)
2407{
2408 return __skb_tx_hash(dev, skb, dev->real_num_tx_queues);
2409}
2410
2411
2412
2413
2414
2415
2416
2417static inline bool netif_is_multiqueue(const struct net_device *dev)
2418{
2419 return dev->num_tx_queues > 1;
2420}
2421
2422int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
2423
2424#ifdef CONFIG_SYSFS
2425int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
2426#else
2427static inline int netif_set_real_num_rx_queues(struct net_device *dev,
2428 unsigned int rxq)
2429{
2430 return 0;
2431}
2432#endif
2433
2434static inline int netif_copy_real_num_queues(struct net_device *to_dev,
2435 const struct net_device *from_dev)
2436{
2437 int err;
2438
2439 err = netif_set_real_num_tx_queues(to_dev,
2440 from_dev->real_num_tx_queues);
2441 if (err)
2442 return err;
2443#ifdef CONFIG_SYSFS
2444 return netif_set_real_num_rx_queues(to_dev,
2445 from_dev->real_num_rx_queues);
2446#else
2447 return 0;
2448#endif
2449}
2450
2451#ifdef CONFIG_SYSFS
2452static inline unsigned int get_netdev_rx_queue_index(
2453 struct netdev_rx_queue *queue)
2454{
2455 struct net_device *dev = queue->dev;
2456 int index = queue - dev->_rx;
2457
2458 BUG_ON(index >= dev->num_rx_queues);
2459 return index;
2460}
2461#endif
2462
2463#define DEFAULT_MAX_NUM_RSS_QUEUES (8)
2464int netif_get_num_default_rss_queues(void);
2465
2466enum skb_free_reason {
2467 SKB_REASON_CONSUMED,
2468 SKB_REASON_DROPPED,
2469};
2470
2471void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
2472void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493static inline void dev_kfree_skb_irq(struct sk_buff *skb)
2494{
2495 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
2496}
2497
2498static inline void dev_consume_skb_irq(struct sk_buff *skb)
2499{
2500 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
2501}
2502
2503static inline void dev_kfree_skb_any(struct sk_buff *skb)
2504{
2505 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
2506}
2507
2508static inline void dev_consume_skb_any(struct sk_buff *skb)
2509{
2510 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
2511}
2512
2513int netif_rx(struct sk_buff *skb);
2514int netif_rx_ni(struct sk_buff *skb);
2515int netif_receive_skb(struct sk_buff *skb);
2516gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
2517void napi_gro_flush(struct napi_struct *napi, bool flush_old);
2518struct sk_buff *napi_get_frags(struct napi_struct *napi);
2519gro_result_t napi_gro_frags(struct napi_struct *napi);
2520struct packet_offload *gro_find_receive_by_type(__be16 type);
2521struct packet_offload *gro_find_complete_by_type(__be16 type);
2522
2523static inline void napi_free_frags(struct napi_struct *napi)
2524{
2525 kfree_skb(napi->skb);
2526 napi->skb = NULL;
2527}
2528
2529int netdev_rx_handler_register(struct net_device *dev,
2530 rx_handler_func_t *rx_handler,
2531 void *rx_handler_data);
2532void netdev_rx_handler_unregister(struct net_device *dev);
2533
2534bool dev_valid_name(const char *name);
2535int dev_ioctl(struct net *net, unsigned int cmd, void __user *);
2536int dev_ethtool(struct net *net, struct ifreq *);
2537unsigned int dev_get_flags(const struct net_device *);
2538int __dev_change_flags(struct net_device *, unsigned int flags);
2539int dev_change_flags(struct net_device *, unsigned int);
2540void __dev_notify_flags(struct net_device *, unsigned int old_flags,
2541 unsigned int gchanges);
2542int dev_change_name(struct net_device *, const char *);
2543int dev_set_alias(struct net_device *, const char *, size_t);
2544int dev_change_net_namespace(struct net_device *, struct net *, const char *);
2545int dev_set_mtu(struct net_device *, int);
2546void dev_set_group(struct net_device *, int);
2547int dev_set_mac_address(struct net_device *, struct sockaddr *);
2548int dev_change_carrier(struct net_device *, bool new_carrier);
2549int dev_get_phys_port_id(struct net_device *dev,
2550 struct netdev_phys_port_id *ppid);
2551int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2552 struct netdev_queue *txq);
2553int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
2554
2555extern int netdev_budget;
2556
2557
2558void netdev_run_todo(void);
2559
2560
2561
2562
2563
2564
2565
2566static inline void dev_put(struct net_device *dev)
2567{
2568 this_cpu_dec(*dev->pcpu_refcnt);
2569}
2570
2571
2572
2573
2574
2575
2576
2577static inline void dev_hold(struct net_device *dev)
2578{
2579 this_cpu_inc(*dev->pcpu_refcnt);
2580}
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591void linkwatch_init_dev(struct net_device *dev);
2592void linkwatch_fire_event(struct net_device *dev);
2593void linkwatch_forget_dev(struct net_device *dev);
2594
2595
2596
2597
2598
2599
2600
2601static inline bool netif_carrier_ok(const struct net_device *dev)
2602{
2603 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
2604}
2605
2606unsigned long dev_trans_start(struct net_device *dev);
2607
2608void __netdev_watchdog_up(struct net_device *dev);
2609
2610void netif_carrier_on(struct net_device *dev);
2611
2612void netif_carrier_off(struct net_device *dev);
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627static inline void netif_dormant_on(struct net_device *dev)
2628{
2629 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
2630 linkwatch_fire_event(dev);
2631}
2632
2633
2634
2635
2636
2637
2638
2639static inline void netif_dormant_off(struct net_device *dev)
2640{
2641 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
2642 linkwatch_fire_event(dev);
2643}
2644
2645
2646
2647
2648
2649
2650
2651static inline bool netif_dormant(const struct net_device *dev)
2652{
2653 return test_bit(__LINK_STATE_DORMANT, &dev->state);
2654}
2655
2656
2657
2658
2659
2660
2661
2662
2663static inline bool netif_oper_up(const struct net_device *dev)
2664{
2665 return (dev->operstate == IF_OPER_UP ||
2666 dev->operstate == IF_OPER_UNKNOWN );
2667}
2668
2669
2670
2671
2672
2673
2674
2675static inline bool netif_device_present(struct net_device *dev)
2676{
2677 return test_bit(__LINK_STATE_PRESENT, &dev->state);
2678}
2679
2680void netif_device_detach(struct net_device *dev);
2681
2682void netif_device_attach(struct net_device *dev);
2683
2684
2685
2686
2687
2688enum {
2689 NETIF_MSG_DRV = 0x0001,
2690 NETIF_MSG_PROBE = 0x0002,
2691 NETIF_MSG_LINK = 0x0004,
2692 NETIF_MSG_TIMER = 0x0008,
2693 NETIF_MSG_IFDOWN = 0x0010,
2694 NETIF_MSG_IFUP = 0x0020,
2695 NETIF_MSG_RX_ERR = 0x0040,
2696 NETIF_MSG_TX_ERR = 0x0080,
2697 NETIF_MSG_TX_QUEUED = 0x0100,
2698 NETIF_MSG_INTR = 0x0200,
2699 NETIF_MSG_TX_DONE = 0x0400,
2700 NETIF_MSG_RX_STATUS = 0x0800,
2701 NETIF_MSG_PKTDATA = 0x1000,
2702 NETIF_MSG_HW = 0x2000,
2703 NETIF_MSG_WOL = 0x4000,
2704};
2705
2706#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
2707#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
2708#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
2709#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
2710#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
2711#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
2712#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
2713#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
2714#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
2715#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
2716#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
2717#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
2718#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
2719#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
2720#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
2721
2722static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
2723{
2724
2725 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
2726 return default_msg_enable_bits;
2727 if (debug_value == 0)
2728 return 0;
2729
2730 return (1 << debug_value) - 1;
2731}
2732
2733static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
2734{
2735 spin_lock(&txq->_xmit_lock);
2736 txq->xmit_lock_owner = cpu;
2737}
2738
2739static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
2740{
2741 spin_lock_bh(&txq->_xmit_lock);
2742 txq->xmit_lock_owner = smp_processor_id();
2743}
2744
2745static inline bool __netif_tx_trylock(struct netdev_queue *txq)
2746{
2747 bool ok = spin_trylock(&txq->_xmit_lock);
2748 if (likely(ok))
2749 txq->xmit_lock_owner = smp_processor_id();
2750 return ok;
2751}
2752
2753static inline void __netif_tx_unlock(struct netdev_queue *txq)
2754{
2755 txq->xmit_lock_owner = -1;
2756 spin_unlock(&txq->_xmit_lock);
2757}
2758
2759static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
2760{
2761 txq->xmit_lock_owner = -1;
2762 spin_unlock_bh(&txq->_xmit_lock);
2763}
2764
2765static inline void txq_trans_update(struct netdev_queue *txq)
2766{
2767 if (txq->xmit_lock_owner != -1)
2768 txq->trans_start = jiffies;
2769}
2770
2771
2772
2773
2774
2775
2776
2777static inline void netif_tx_lock(struct net_device *dev)
2778{
2779 unsigned int i;
2780 int cpu;
2781
2782 spin_lock(&dev->tx_global_lock);
2783 cpu = smp_processor_id();
2784 for (i = 0; i < dev->num_tx_queues; i++) {
2785 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2786
2787
2788
2789
2790
2791
2792
2793 __netif_tx_lock(txq, cpu);
2794 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
2795 __netif_tx_unlock(txq);
2796 }
2797}
2798
2799static inline void netif_tx_lock_bh(struct net_device *dev)
2800{
2801 local_bh_disable();
2802 netif_tx_lock(dev);
2803}
2804
2805static inline void netif_tx_unlock(struct net_device *dev)
2806{
2807 unsigned int i;
2808
2809 for (i = 0; i < dev->num_tx_queues; i++) {
2810 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2811
2812
2813
2814
2815
2816 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
2817 netif_schedule_queue(txq);
2818 }
2819 spin_unlock(&dev->tx_global_lock);
2820}
2821
2822static inline void netif_tx_unlock_bh(struct net_device *dev)
2823{
2824 netif_tx_unlock(dev);
2825 local_bh_enable();
2826}
2827
2828#define HARD_TX_LOCK(dev, txq, cpu) { \
2829 if ((dev->features & NETIF_F_LLTX) == 0) { \
2830 __netif_tx_lock(txq, cpu); \
2831 } \
2832}
2833
2834#define HARD_TX_UNLOCK(dev, txq) { \
2835 if ((dev->features & NETIF_F_LLTX) == 0) { \
2836 __netif_tx_unlock(txq); \
2837 } \
2838}
2839
2840static inline void netif_tx_disable(struct net_device *dev)
2841{
2842 unsigned int i;
2843 int cpu;
2844
2845 local_bh_disable();
2846 cpu = smp_processor_id();
2847 for (i = 0; i < dev->num_tx_queues; i++) {
2848 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
2849
2850 __netif_tx_lock(txq, cpu);
2851 netif_tx_stop_queue(txq);
2852 __netif_tx_unlock(txq);
2853 }
2854 local_bh_enable();
2855}
2856
2857static inline void netif_addr_lock(struct net_device *dev)
2858{
2859 spin_lock(&dev->addr_list_lock);
2860}
2861
2862static inline void netif_addr_lock_nested(struct net_device *dev)
2863{
2864 spin_lock_nested(&dev->addr_list_lock, SINGLE_DEPTH_NESTING);
2865}
2866
2867static inline void netif_addr_lock_bh(struct net_device *dev)
2868{
2869 spin_lock_bh(&dev->addr_list_lock);
2870}
2871
2872static inline void netif_addr_unlock(struct net_device *dev)
2873{
2874 spin_unlock(&dev->addr_list_lock);
2875}
2876
2877static inline void netif_addr_unlock_bh(struct net_device *dev)
2878{
2879 spin_unlock_bh(&dev->addr_list_lock);
2880}
2881
2882
2883
2884
2885
2886#define for_each_dev_addr(dev, ha) \
2887 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
2888
2889
2890
2891void ether_setup(struct net_device *dev);
2892
2893
2894struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
2895 void (*setup)(struct net_device *),
2896 unsigned int txqs, unsigned int rxqs);
2897#define alloc_netdev(sizeof_priv, name, setup) \
2898 alloc_netdev_mqs(sizeof_priv, name, setup, 1, 1)
2899
2900#define alloc_netdev_mq(sizeof_priv, name, setup, count) \
2901 alloc_netdev_mqs(sizeof_priv, name, setup, count, count)
2902
2903int register_netdev(struct net_device *dev);
2904void unregister_netdev(struct net_device *dev);
2905
2906
2907int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
2908 struct netdev_hw_addr_list *from_list, int addr_len);
2909void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
2910 struct netdev_hw_addr_list *from_list, int addr_len);
2911void __hw_addr_init(struct netdev_hw_addr_list *list);
2912
2913
2914int dev_addr_add(struct net_device *dev, const unsigned char *addr,
2915 unsigned char addr_type);
2916int dev_addr_del(struct net_device *dev, const unsigned char *addr,
2917 unsigned char addr_type);
2918void dev_addr_flush(struct net_device *dev);
2919int dev_addr_init(struct net_device *dev);
2920
2921
2922int dev_uc_add(struct net_device *dev, const unsigned char *addr);
2923int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
2924int dev_uc_del(struct net_device *dev, const unsigned char *addr);
2925int dev_uc_sync(struct net_device *to, struct net_device *from);
2926int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
2927void dev_uc_unsync(struct net_device *to, struct net_device *from);
2928void dev_uc_flush(struct net_device *dev);
2929void dev_uc_init(struct net_device *dev);
2930
2931
2932int dev_mc_add(struct net_device *dev, const unsigned char *addr);
2933int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
2934int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
2935int dev_mc_del(struct net_device *dev, const unsigned char *addr);
2936int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
2937int dev_mc_sync(struct net_device *to, struct net_device *from);
2938int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
2939void dev_mc_unsync(struct net_device *to, struct net_device *from);
2940void dev_mc_flush(struct net_device *dev);
2941void dev_mc_init(struct net_device *dev);
2942
2943
2944void dev_set_rx_mode(struct net_device *dev);
2945void __dev_set_rx_mode(struct net_device *dev);
2946int dev_set_promiscuity(struct net_device *dev, int inc);
2947int dev_set_allmulti(struct net_device *dev, int inc);
2948void netdev_state_change(struct net_device *dev);
2949void netdev_notify_peers(struct net_device *dev);
2950void netdev_features_change(struct net_device *dev);
2951
2952void dev_load(struct net *net, const char *name);
2953struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
2954 struct rtnl_link_stats64 *storage);
2955void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
2956 const struct net_device_stats *netdev_stats);
2957
2958extern int netdev_max_backlog;
2959extern int netdev_tstamp_prequeue;
2960extern int weight_p;
2961extern int bpf_jit_enable;
2962
2963bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
2964struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
2965 struct list_head **iter);
2966
2967
2968#define netdev_for_each_all_upper_dev_rcu(dev, updev, iter) \
2969 for (iter = &(dev)->all_adj_list.upper, \
2970 updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)); \
2971 updev; \
2972 updev = netdev_all_upper_get_next_dev_rcu(dev, &(iter)))
2973
2974void *netdev_lower_get_next_private(struct net_device *dev,
2975 struct list_head **iter);
2976void *netdev_lower_get_next_private_rcu(struct net_device *dev,
2977 struct list_head **iter);
2978
2979#define netdev_for_each_lower_private(dev, priv, iter) \
2980 for (iter = (dev)->adj_list.lower.next, \
2981 priv = netdev_lower_get_next_private(dev, &(iter)); \
2982 priv; \
2983 priv = netdev_lower_get_next_private(dev, &(iter)))
2984
2985#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
2986 for (iter = &(dev)->adj_list.lower, \
2987 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
2988 priv; \
2989 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
2990
2991void *netdev_adjacent_get_private(struct list_head *adj_list);
2992void *netdev_lower_get_first_private_rcu(struct net_device *dev);
2993struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
2994struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
2995int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev);
2996int netdev_master_upper_dev_link(struct net_device *dev,
2997 struct net_device *upper_dev);
2998int netdev_master_upper_dev_link_private(struct net_device *dev,
2999 struct net_device *upper_dev,
3000 void *private);
3001void netdev_upper_dev_unlink(struct net_device *dev,
3002 struct net_device *upper_dev);
3003void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
3004void *netdev_lower_dev_get_private(struct net_device *dev,
3005 struct net_device *lower_dev);
3006int skb_checksum_help(struct sk_buff *skb);
3007struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3008 netdev_features_t features, bool tx_path);
3009struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3010 netdev_features_t features);
3011
3012static inline
3013struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
3014{
3015 return __skb_gso_segment(skb, features, true);
3016}
3017__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
3018
3019static inline bool can_checksum_protocol(netdev_features_t features,
3020 __be16 protocol)
3021{
3022 return ((features & NETIF_F_GEN_CSUM) ||
3023 ((features & NETIF_F_V4_CSUM) &&
3024 protocol == htons(ETH_P_IP)) ||
3025 ((features & NETIF_F_V6_CSUM) &&
3026 protocol == htons(ETH_P_IPV6)) ||
3027 ((features & NETIF_F_FCOE_CRC) &&
3028 protocol == htons(ETH_P_FCOE)));
3029}
3030
3031#ifdef CONFIG_BUG
3032void netdev_rx_csum_fault(struct net_device *dev);
3033#else
3034static inline void netdev_rx_csum_fault(struct net_device *dev)
3035{
3036}
3037#endif
3038
3039void net_enable_timestamp(void);
3040void net_disable_timestamp(void);
3041
3042#ifdef CONFIG_PROC_FS
3043int __init dev_proc_init(void);
3044#else
3045#define dev_proc_init() 0
3046#endif
3047
3048int netdev_class_create_file_ns(struct class_attribute *class_attr,
3049 const void *ns);
3050void netdev_class_remove_file_ns(struct class_attribute *class_attr,
3051 const void *ns);
3052
3053static inline int netdev_class_create_file(struct class_attribute *class_attr)
3054{
3055 return netdev_class_create_file_ns(class_attr, NULL);
3056}
3057
3058static inline void netdev_class_remove_file(struct class_attribute *class_attr)
3059{
3060 netdev_class_remove_file_ns(class_attr, NULL);
3061}
3062
3063extern struct kobj_ns_type_operations net_ns_type_operations;
3064
3065const char *netdev_drivername(const struct net_device *dev);
3066
3067void linkwatch_run_queue(void);
3068
3069static inline netdev_features_t netdev_get_wanted_features(
3070 struct net_device *dev)
3071{
3072 return (dev->features & ~dev->hw_features) | dev->wanted_features;
3073}
3074netdev_features_t netdev_increment_features(netdev_features_t all,
3075 netdev_features_t one, netdev_features_t mask);
3076
3077
3078
3079
3080
3081static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
3082 netdev_features_t mask)
3083{
3084 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
3085}
3086
3087int __netdev_update_features(struct net_device *dev);
3088void netdev_update_features(struct net_device *dev);
3089void netdev_change_features(struct net_device *dev);
3090
3091void netif_stacked_transfer_operstate(const struct net_device *rootdev,
3092 struct net_device *dev);
3093
3094netdev_features_t netif_skb_dev_features(struct sk_buff *skb,
3095 const struct net_device *dev);
3096static inline netdev_features_t netif_skb_features(struct sk_buff *skb)
3097{
3098 return netif_skb_dev_features(skb, skb->dev);
3099}
3100
3101static inline bool net_gso_ok(netdev_features_t features, int gso_type)
3102{
3103 netdev_features_t feature = gso_type << NETIF_F_GSO_SHIFT;
3104
3105
3106 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
3107 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_UFO >> NETIF_F_GSO_SHIFT));
3108 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
3109 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
3110 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
3111 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
3112
3113 return (features & feature) == feature;
3114}
3115
3116static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
3117{
3118 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
3119 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
3120}
3121
3122static inline bool netif_needs_gso(struct sk_buff *skb,
3123 netdev_features_t features)
3124{
3125 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
3126 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
3127 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
3128}
3129
3130static inline void netif_set_gso_max_size(struct net_device *dev,
3131 unsigned int size)
3132{
3133 dev->gso_max_size = size;
3134}
3135
3136static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
3137 int pulled_hlen, u16 mac_offset,
3138 int mac_len)
3139{
3140 skb->protocol = protocol;
3141 skb->encapsulation = 1;
3142 skb_push(skb, pulled_hlen);
3143 skb_reset_transport_header(skb);
3144 skb->mac_header = mac_offset;
3145 skb->network_header = skb->mac_header + mac_len;
3146 skb->mac_len = mac_len;
3147}
3148
3149static inline bool netif_is_macvlan(struct net_device *dev)
3150{
3151 return dev->priv_flags & IFF_MACVLAN;
3152}
3153
3154static inline bool netif_is_bond_master(struct net_device *dev)
3155{
3156 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
3157}
3158
3159static inline bool netif_is_bond_slave(struct net_device *dev)
3160{
3161 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
3162}
3163
3164static inline bool netif_supports_nofcs(struct net_device *dev)
3165{
3166 return dev->priv_flags & IFF_SUPP_NOFCS;
3167}
3168
3169extern struct pernet_operations __net_initdata loopback_net_ops;
3170
3171
3172
3173
3174
3175static inline const char *netdev_name(const struct net_device *dev)
3176{
3177 if (dev->reg_state != NETREG_REGISTERED)
3178 return "(unregistered net_device)";
3179 return dev->name;
3180}
3181
3182__printf(3, 4)
3183int netdev_printk(const char *level, const struct net_device *dev,
3184 const char *format, ...);
3185__printf(2, 3)
3186int netdev_emerg(const struct net_device *dev, const char *format, ...);
3187__printf(2, 3)
3188int netdev_alert(const struct net_device *dev, const char *format, ...);
3189__printf(2, 3)
3190int netdev_crit(const struct net_device *dev, const char *format, ...);
3191__printf(2, 3)
3192int netdev_err(const struct net_device *dev, const char *format, ...);
3193__printf(2, 3)
3194int netdev_warn(const struct net_device *dev, const char *format, ...);
3195__printf(2, 3)
3196int netdev_notice(const struct net_device *dev, const char *format, ...);
3197__printf(2, 3)
3198int netdev_info(const struct net_device *dev, const char *format, ...);
3199
3200#define MODULE_ALIAS_NETDEV(device) \
3201 MODULE_ALIAS("netdev-" device)
3202
3203#if defined(CONFIG_DYNAMIC_DEBUG)
3204#define netdev_dbg(__dev, format, args...) \
3205do { \
3206 dynamic_netdev_dbg(__dev, format, ##args); \
3207} while (0)
3208#elif defined(DEBUG)
3209#define netdev_dbg(__dev, format, args...) \
3210 netdev_printk(KERN_DEBUG, __dev, format, ##args)
3211#else
3212#define netdev_dbg(__dev, format, args...) \
3213({ \
3214 if (0) \
3215 netdev_printk(KERN_DEBUG, __dev, format, ##args); \
3216 0; \
3217})
3218#endif
3219
3220#if defined(VERBOSE_DEBUG)
3221#define netdev_vdbg netdev_dbg
3222#else
3223
3224#define netdev_vdbg(dev, format, args...) \
3225({ \
3226 if (0) \
3227 netdev_printk(KERN_DEBUG, dev, format, ##args); \
3228 0; \
3229})
3230#endif
3231
3232
3233
3234
3235
3236
3237#define netdev_WARN(dev, format, args...) \
3238 WARN(1, "netdevice: %s\n" format, netdev_name(dev), ##args)
3239
3240
3241
3242#define netif_printk(priv, type, level, dev, fmt, args...) \
3243do { \
3244 if (netif_msg_##type(priv)) \
3245 netdev_printk(level, (dev), fmt, ##args); \
3246} while (0)
3247
3248#define netif_level(level, priv, type, dev, fmt, args...) \
3249do { \
3250 if (netif_msg_##type(priv)) \
3251 netdev_##level(dev, fmt, ##args); \
3252} while (0)
3253
3254#define netif_emerg(priv, type, dev, fmt, args...) \
3255 netif_level(emerg, priv, type, dev, fmt, ##args)
3256#define netif_alert(priv, type, dev, fmt, args...) \
3257 netif_level(alert, priv, type, dev, fmt, ##args)
3258#define netif_crit(priv, type, dev, fmt, args...) \
3259 netif_level(crit, priv, type, dev, fmt, ##args)
3260#define netif_err(priv, type, dev, fmt, args...) \
3261 netif_level(err, priv, type, dev, fmt, ##args)
3262#define netif_warn(priv, type, dev, fmt, args...) \
3263 netif_level(warn, priv, type, dev, fmt, ##args)
3264#define netif_notice(priv, type, dev, fmt, args...) \
3265 netif_level(notice, priv, type, dev, fmt, ##args)
3266#define netif_info(priv, type, dev, fmt, args...) \
3267 netif_level(info, priv, type, dev, fmt, ##args)
3268
3269#if defined(CONFIG_DYNAMIC_DEBUG)
3270#define netif_dbg(priv, type, netdev, format, args...) \
3271do { \
3272 if (netif_msg_##type(priv)) \
3273 dynamic_netdev_dbg(netdev, format, ##args); \
3274} while (0)
3275#elif defined(DEBUG)
3276#define netif_dbg(priv, type, dev, format, args...) \
3277 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
3278#else
3279#define netif_dbg(priv, type, dev, format, args...) \
3280({ \
3281 if (0) \
3282 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
3283 0; \
3284})
3285#endif
3286
3287#if defined(VERBOSE_DEBUG)
3288#define netif_vdbg netif_dbg
3289#else
3290#define netif_vdbg(priv, type, dev, format, args...) \
3291({ \
3292 if (0) \
3293 netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
3294 0; \
3295})
3296#endif
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325#define PTYPE_HASH_SIZE (16)
3326#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
3327
3328#endif
3329