1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/module.h>
28#include <linux/kernel.h>
29#include <linux/netdevice.h>
30#include <linux/etherdevice.h>
31#include <linux/init.h>
32#include <linux/interrupt.h>
33#include <linux/moduleparam.h>
34#include <net/pkt_sched.h>
35#include <net/net_namespace.h>
36
37#define TX_Q_LIMIT 32
38struct ifb_q_private {
39 struct net_device *dev;
40 struct tasklet_struct ifb_tasklet;
41 int tasklet_pending;
42 int txqnum;
43 struct sk_buff_head rq;
44 u64 rx_packets;
45 u64 rx_bytes;
46 struct u64_stats_sync rsync;
47
48 struct u64_stats_sync tsync;
49 u64 tx_packets;
50 u64 tx_bytes;
51 struct sk_buff_head tq;
52} ____cacheline_aligned_in_smp;
53
54struct ifb_dev_private {
55 struct ifb_q_private *tx_private;
56};
57
58static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
59static int ifb_open(struct net_device *dev);
60static int ifb_close(struct net_device *dev);
61
62static void ifb_ri_tasklet(unsigned long _txp)
63{
64 struct ifb_q_private *txp = (struct ifb_q_private *)_txp;
65 struct netdev_queue *txq;
66 struct sk_buff *skb;
67
68 txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
69 skb = skb_peek(&txp->tq);
70 if (!skb) {
71 if (!__netif_tx_trylock(txq))
72 goto resched;
73 skb_queue_splice_tail_init(&txp->rq, &txp->tq);
74 __netif_tx_unlock(txq);
75 }
76
77 while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
78 skb->redirected = 0;
79 skb->tc_skip_classify = 1;
80
81 u64_stats_update_begin(&txp->tsync);
82 txp->tx_packets++;
83 txp->tx_bytes += skb->len;
84 u64_stats_update_end(&txp->tsync);
85
86 rcu_read_lock();
87 skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
88 if (!skb->dev) {
89 rcu_read_unlock();
90 dev_kfree_skb(skb);
91 txp->dev->stats.tx_dropped++;
92 if (skb_queue_len(&txp->tq) != 0)
93 goto resched;
94 break;
95 }
96 rcu_read_unlock();
97 skb->skb_iif = txp->dev->ifindex;
98
99 if (!skb->from_ingress) {
100 dev_queue_xmit(skb);
101 } else {
102 skb_pull_rcsum(skb, skb->mac_len);
103 netif_receive_skb(skb);
104 }
105 }
106
107 if (__netif_tx_trylock(txq)) {
108 skb = skb_peek(&txp->rq);
109 if (!skb) {
110 txp->tasklet_pending = 0;
111 if (netif_tx_queue_stopped(txq))
112 netif_tx_wake_queue(txq);
113 } else {
114 __netif_tx_unlock(txq);
115 goto resched;
116 }
117 __netif_tx_unlock(txq);
118 } else {
119resched:
120 txp->tasklet_pending = 1;
121 tasklet_schedule(&txp->ifb_tasklet);
122 }
123
124}
125
126static void ifb_stats64(struct net_device *dev,
127 struct rtnl_link_stats64 *stats)
128{
129 struct ifb_dev_private *dp = netdev_priv(dev);
130 struct ifb_q_private *txp = dp->tx_private;
131 unsigned int start;
132 u64 packets, bytes;
133 int i;
134
135 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
136 do {
137 start = u64_stats_fetch_begin_irq(&txp->rsync);
138 packets = txp->rx_packets;
139 bytes = txp->rx_bytes;
140 } while (u64_stats_fetch_retry_irq(&txp->rsync, start));
141 stats->rx_packets += packets;
142 stats->rx_bytes += bytes;
143
144 do {
145 start = u64_stats_fetch_begin_irq(&txp->tsync);
146 packets = txp->tx_packets;
147 bytes = txp->tx_bytes;
148 } while (u64_stats_fetch_retry_irq(&txp->tsync, start));
149 stats->tx_packets += packets;
150 stats->tx_bytes += bytes;
151 }
152 stats->rx_dropped = dev->stats.rx_dropped;
153 stats->tx_dropped = dev->stats.tx_dropped;
154}
155
156static int ifb_dev_init(struct net_device *dev)
157{
158 struct ifb_dev_private *dp = netdev_priv(dev);
159 struct ifb_q_private *txp;
160 int i;
161
162 txp = kcalloc(dev->num_tx_queues, sizeof(*txp), GFP_KERNEL);
163 if (!txp)
164 return -ENOMEM;
165 dp->tx_private = txp;
166 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
167 txp->txqnum = i;
168 txp->dev = dev;
169 __skb_queue_head_init(&txp->rq);
170 __skb_queue_head_init(&txp->tq);
171 u64_stats_init(&txp->rsync);
172 u64_stats_init(&txp->tsync);
173 tasklet_init(&txp->ifb_tasklet, ifb_ri_tasklet,
174 (unsigned long)txp);
175 netif_tx_start_queue(netdev_get_tx_queue(dev, i));
176 }
177 return 0;
178}
179
180static const struct net_device_ops ifb_netdev_ops = {
181 .ndo_open = ifb_open,
182 .ndo_stop = ifb_close,
183 .ndo_get_stats64 = ifb_stats64,
184 .ndo_start_xmit = ifb_xmit,
185 .ndo_validate_addr = eth_validate_addr,
186 .ndo_init = ifb_dev_init,
187};
188
189#define IFB_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_FRAGLIST | \
190 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \
191 NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX | \
192 NETIF_F_HW_VLAN_STAG_TX)
193
194static void ifb_dev_free(struct net_device *dev)
195{
196 struct ifb_dev_private *dp = netdev_priv(dev);
197 struct ifb_q_private *txp = dp->tx_private;
198 int i;
199
200 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
201 tasklet_kill(&txp->ifb_tasklet);
202 __skb_queue_purge(&txp->rq);
203 __skb_queue_purge(&txp->tq);
204 }
205 kfree(dp->tx_private);
206}
207
208static void ifb_setup(struct net_device *dev)
209{
210
211 dev->netdev_ops = &ifb_netdev_ops;
212
213
214 ether_setup(dev);
215 dev->tx_queue_len = TX_Q_LIMIT;
216
217 dev->features |= IFB_FEATURES;
218 dev->hw_features |= dev->features;
219 dev->hw_enc_features |= dev->features;
220 dev->vlan_features |= IFB_FEATURES & ~(NETIF_F_HW_VLAN_CTAG_TX |
221 NETIF_F_HW_VLAN_STAG_TX);
222
223 dev->flags |= IFF_NOARP;
224 dev->flags &= ~IFF_MULTICAST;
225 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
226 netif_keep_dst(dev);
227 eth_hw_addr_random(dev);
228 dev->needs_free_netdev = true;
229 dev->priv_destructor = ifb_dev_free;
230
231 dev->min_mtu = 0;
232 dev->max_mtu = 0;
233}
234
235static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
236{
237 struct ifb_dev_private *dp = netdev_priv(dev);
238 struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);
239
240 u64_stats_update_begin(&txp->rsync);
241 txp->rx_packets++;
242 txp->rx_bytes += skb->len;
243 u64_stats_update_end(&txp->rsync);
244
245 if (!skb->redirected || !skb->skb_iif) {
246 dev_kfree_skb(skb);
247 dev->stats.rx_dropped++;
248 return NETDEV_TX_OK;
249 }
250
251 if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
252 netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));
253
254 __skb_queue_tail(&txp->rq, skb);
255 if (!txp->tasklet_pending) {
256 txp->tasklet_pending = 1;
257 tasklet_schedule(&txp->ifb_tasklet);
258 }
259
260 return NETDEV_TX_OK;
261}
262
263static int ifb_close(struct net_device *dev)
264{
265 netif_tx_stop_all_queues(dev);
266 return 0;
267}
268
269static int ifb_open(struct net_device *dev)
270{
271 netif_tx_start_all_queues(dev);
272 return 0;
273}
274
275static int ifb_validate(struct nlattr *tb[], struct nlattr *data[],
276 struct netlink_ext_ack *extack)
277{
278 if (tb[IFLA_ADDRESS]) {
279 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
280 return -EINVAL;
281 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
282 return -EADDRNOTAVAIL;
283 }
284 return 0;
285}
286
287static struct rtnl_link_ops ifb_link_ops __read_mostly = {
288 .kind = "ifb",
289 .priv_size = sizeof(struct ifb_dev_private),
290 .setup = ifb_setup,
291 .validate = ifb_validate,
292};
293
294
295
296
297
298static int numifbs = 2;
299module_param(numifbs, int, 0);
300MODULE_PARM_DESC(numifbs, "Number of ifb devices");
301
302static int __init ifb_init_one(int index)
303{
304 struct net_device *dev_ifb;
305 int err;
306
307 dev_ifb = alloc_netdev(sizeof(struct ifb_dev_private), "ifb%d",
308 NET_NAME_UNKNOWN, ifb_setup);
309
310 if (!dev_ifb)
311 return -ENOMEM;
312
313 dev_ifb->rtnl_link_ops = &ifb_link_ops;
314 err = register_netdevice(dev_ifb);
315 if (err < 0)
316 goto err;
317
318 return 0;
319
320err:
321 free_netdev(dev_ifb);
322 return err;
323}
324
325static int __init ifb_init_module(void)
326{
327 int i, err;
328
329 down_write(&pernet_ops_rwsem);
330 rtnl_lock();
331 err = __rtnl_link_register(&ifb_link_ops);
332 if (err < 0)
333 goto out;
334
335 for (i = 0; i < numifbs && !err; i++) {
336 err = ifb_init_one(i);
337 cond_resched();
338 }
339 if (err)
340 __rtnl_link_unregister(&ifb_link_ops);
341
342out:
343 rtnl_unlock();
344 up_write(&pernet_ops_rwsem);
345
346 return err;
347}
348
349static void __exit ifb_cleanup_module(void)
350{
351 rtnl_link_unregister(&ifb_link_ops);
352}
353
354module_init(ifb_init_module);
355module_exit(ifb_cleanup_module);
356MODULE_LICENSE("GPL");
357MODULE_AUTHOR("Jamal Hadi Salim");
358MODULE_ALIAS_RTNL_LINK("ifb");
359
