1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/module.h>
28#include <linux/kernel.h>
29#include <linux/netdevice.h>
30#include <linux/etherdevice.h>
31#include <linux/init.h>
32#include <linux/interrupt.h>
33#include <linux/moduleparam.h>
34#include <net/pkt_sched.h>
35#include <net/net_namespace.h>
36
37#define TX_Q_LIMIT 32
38struct ifb_q_private {
39 struct net_device *dev;
40 struct tasklet_struct ifb_tasklet;
41 int tasklet_pending;
42 int txqnum;
43 struct sk_buff_head rq;
44 u64 rx_packets;
45 u64 rx_bytes;
46 struct u64_stats_sync rsync;
47
48 struct u64_stats_sync tsync;
49 u64 tx_packets;
50 u64 tx_bytes;
51 struct sk_buff_head tq;
52} ____cacheline_aligned_in_smp;
53
54struct ifb_dev_private {
55 struct ifb_q_private *tx_private;
56};
57
58static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
59static int ifb_open(struct net_device *dev);
60static int ifb_close(struct net_device *dev);
61
62static void ifb_ri_tasklet(struct tasklet_struct *t)
63{
64 struct ifb_q_private *txp = from_tasklet(txp, t, ifb_tasklet);
65 struct netdev_queue *txq;
66 struct sk_buff *skb;
67
68 txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
69 skb = skb_peek(&txp->tq);
70 if (!skb) {
71 if (!__netif_tx_trylock(txq))
72 goto resched;
73 skb_queue_splice_tail_init(&txp->rq, &txp->tq);
74 __netif_tx_unlock(txq);
75 }
76
77 while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
78 skb->redirected = 0;
79 skb->tc_skip_classify = 1;
80
81 u64_stats_update_begin(&txp->tsync);
82 txp->tx_packets++;
83 txp->tx_bytes += skb->len;
84 u64_stats_update_end(&txp->tsync);
85
86 rcu_read_lock();
87 skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
88 if (!skb->dev) {
89 rcu_read_unlock();
90 dev_kfree_skb(skb);
91 txp->dev->stats.tx_dropped++;
92 if (skb_queue_len(&txp->tq) != 0)
93 goto resched;
94 break;
95 }
96 rcu_read_unlock();
97 skb->skb_iif = txp->dev->ifindex;
98
99 if (!skb->from_ingress) {
100 dev_queue_xmit(skb);
101 } else {
102 skb_pull_rcsum(skb, skb->mac_len);
103 netif_receive_skb(skb);
104 }
105 }
106
107 if (__netif_tx_trylock(txq)) {
108 skb = skb_peek(&txp->rq);
109 if (!skb) {
110 txp->tasklet_pending = 0;
111 if (netif_tx_queue_stopped(txq))
112 netif_tx_wake_queue(txq);
113 } else {
114 __netif_tx_unlock(txq);
115 goto resched;
116 }
117 __netif_tx_unlock(txq);
118 } else {
119resched:
120 txp->tasklet_pending = 1;
121 tasklet_schedule(&txp->ifb_tasklet);
122 }
123
124}
125
126static void ifb_stats64(struct net_device *dev,
127 struct rtnl_link_stats64 *stats)
128{
129 struct ifb_dev_private *dp = netdev_priv(dev);
130 struct ifb_q_private *txp = dp->tx_private;
131 unsigned int start;
132 u64 packets, bytes;
133 int i;
134
135 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
136 do {
137 start = u64_stats_fetch_begin_irq(&txp->rsync);
138 packets = txp->rx_packets;
139 bytes = txp->rx_bytes;
140 } while (u64_stats_fetch_retry_irq(&txp->rsync, start));
141 stats->rx_packets += packets;
142 stats->rx_bytes += bytes;
143
144 do {
145 start = u64_stats_fetch_begin_irq(&txp->tsync);
146 packets = txp->tx_packets;
147 bytes = txp->tx_bytes;
148 } while (u64_stats_fetch_retry_irq(&txp->tsync, start));
149 stats->tx_packets += packets;
150 stats->tx_bytes += bytes;
151 }
152 stats->rx_dropped = dev->stats.rx_dropped;
153 stats->tx_dropped = dev->stats.tx_dropped;
154}
155
156static int ifb_dev_init(struct net_device *dev)
157{
158 struct ifb_dev_private *dp = netdev_priv(dev);
159 struct ifb_q_private *txp;
160 int i;
161
162 txp = kcalloc(dev->num_tx_queues, sizeof(*txp), GFP_KERNEL);
163 if (!txp)
164 return -ENOMEM;
165 dp->tx_private = txp;
166 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
167 txp->txqnum = i;
168 txp->dev = dev;
169 __skb_queue_head_init(&txp->rq);
170 __skb_queue_head_init(&txp->tq);
171 u64_stats_init(&txp->rsync);
172 u64_stats_init(&txp->tsync);
173 tasklet_setup(&txp->ifb_tasklet, ifb_ri_tasklet);
174 netif_tx_start_queue(netdev_get_tx_queue(dev, i));
175 }
176 return 0;
177}
178
179static const struct net_device_ops ifb_netdev_ops = {
180 .ndo_open = ifb_open,
181 .ndo_stop = ifb_close,
182 .ndo_get_stats64 = ifb_stats64,
183 .ndo_start_xmit = ifb_xmit,
184 .ndo_validate_addr = eth_validate_addr,
185 .ndo_init = ifb_dev_init,
186};
187
188#define IFB_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_FRAGLIST | \
189 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \
190 NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX | \
191 NETIF_F_HW_VLAN_STAG_TX)
192
193static void ifb_dev_free(struct net_device *dev)
194{
195 struct ifb_dev_private *dp = netdev_priv(dev);
196 struct ifb_q_private *txp = dp->tx_private;
197 int i;
198
199 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
200 tasklet_kill(&txp->ifb_tasklet);
201 __skb_queue_purge(&txp->rq);
202 __skb_queue_purge(&txp->tq);
203 }
204 kfree(dp->tx_private);
205}
206
207static void ifb_setup(struct net_device *dev)
208{
209
210 dev->netdev_ops = &ifb_netdev_ops;
211
212
213 ether_setup(dev);
214 dev->tx_queue_len = TX_Q_LIMIT;
215
216 dev->features |= IFB_FEATURES;
217 dev->hw_features |= dev->features;
218 dev->hw_enc_features |= dev->features;
219 dev->vlan_features |= IFB_FEATURES & ~(NETIF_F_HW_VLAN_CTAG_TX |
220 NETIF_F_HW_VLAN_STAG_TX);
221
222 dev->flags |= IFF_NOARP;
223 dev->flags &= ~IFF_MULTICAST;
224 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
225 netif_keep_dst(dev);
226 eth_hw_addr_random(dev);
227 dev->needs_free_netdev = true;
228 dev->priv_destructor = ifb_dev_free;
229
230 dev->min_mtu = 0;
231 dev->max_mtu = 0;
232}
233
234static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
235{
236 struct ifb_dev_private *dp = netdev_priv(dev);
237 struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);
238
239 u64_stats_update_begin(&txp->rsync);
240 txp->rx_packets++;
241 txp->rx_bytes += skb->len;
242 u64_stats_update_end(&txp->rsync);
243
244 if (!skb->redirected || !skb->skb_iif) {
245 dev_kfree_skb(skb);
246 dev->stats.rx_dropped++;
247 return NETDEV_TX_OK;
248 }
249
250 if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
251 netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));
252
253 __skb_queue_tail(&txp->rq, skb);
254 if (!txp->tasklet_pending) {
255 txp->tasklet_pending = 1;
256 tasklet_schedule(&txp->ifb_tasklet);
257 }
258
259 return NETDEV_TX_OK;
260}
261
262static int ifb_close(struct net_device *dev)
263{
264 netif_tx_stop_all_queues(dev);
265 return 0;
266}
267
268static int ifb_open(struct net_device *dev)
269{
270 netif_tx_start_all_queues(dev);
271 return 0;
272}
273
274static int ifb_validate(struct nlattr *tb[], struct nlattr *data[],
275 struct netlink_ext_ack *extack)
276{
277 if (tb[IFLA_ADDRESS]) {
278 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
279 return -EINVAL;
280 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
281 return -EADDRNOTAVAIL;
282 }
283 return 0;
284}
285
286static struct rtnl_link_ops ifb_link_ops __read_mostly = {
287 .kind = "ifb",
288 .priv_size = sizeof(struct ifb_dev_private),
289 .setup = ifb_setup,
290 .validate = ifb_validate,
291};
292
293
294
295
296
297static int numifbs = 2;
298module_param(numifbs, int, 0);
299MODULE_PARM_DESC(numifbs, "Number of ifb devices");
300
301static int __init ifb_init_one(int index)
302{
303 struct net_device *dev_ifb;
304 int err;
305
306 dev_ifb = alloc_netdev(sizeof(struct ifb_dev_private), "ifb%d",
307 NET_NAME_UNKNOWN, ifb_setup);
308
309 if (!dev_ifb)
310 return -ENOMEM;
311
312 dev_ifb->rtnl_link_ops = &ifb_link_ops;
313 err = register_netdevice(dev_ifb);
314 if (err < 0)
315 goto err;
316
317 return 0;
318
319err:
320 free_netdev(dev_ifb);
321 return err;
322}
323
324static int __init ifb_init_module(void)
325{
326 int i, err;
327
328 down_write(&pernet_ops_rwsem);
329 rtnl_lock();
330 err = __rtnl_link_register(&ifb_link_ops);
331 if (err < 0)
332 goto out;
333
334 for (i = 0; i < numifbs && !err; i++) {
335 err = ifb_init_one(i);
336 cond_resched();
337 }
338 if (err)
339 __rtnl_link_unregister(&ifb_link_ops);
340
341out:
342 rtnl_unlock();
343 up_write(&pernet_ops_rwsem);
344
345 return err;
346}
347
348static void __exit ifb_cleanup_module(void)
349{
350 rtnl_link_unregister(&ifb_link_ops);
351}
352
353module_init(ifb_init_module);
354module_exit(ifb_cleanup_module);
355MODULE_LICENSE("GPL");
356MODULE_AUTHOR("Jamal Hadi Salim");
357MODULE_ALIAS_RTNL_LINK("ifb");
358
