1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <linux/module.h>
31#include <linux/kernel.h>
32#include <linux/netdevice.h>
33#include <linux/etherdevice.h>
34#include <linux/init.h>
35#include <linux/interrupt.h>
36#include <linux/moduleparam.h>
37#include <net/pkt_sched.h>
38#include <net/net_namespace.h>
39
40#define TX_Q_LIMIT 32
41struct ifb_q_private {
42 struct net_device *dev;
43 struct tasklet_struct ifb_tasklet;
44 int tasklet_pending;
45 int txqnum;
46 struct sk_buff_head rq;
47 u64 rx_packets;
48 u64 rx_bytes;
49 struct u64_stats_sync rsync;
50
51 struct u64_stats_sync tsync;
52 u64 tx_packets;
53 u64 tx_bytes;
54 struct sk_buff_head tq;
55} ____cacheline_aligned_in_smp;
56
57struct ifb_dev_private {
58 struct ifb_q_private *tx_private;
59};
60
61static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
62static int ifb_open(struct net_device *dev);
63static int ifb_close(struct net_device *dev);
64
65static void ifb_ri_tasklet(unsigned long _txp)
66{
67 struct ifb_q_private *txp = (struct ifb_q_private *)_txp;
68 struct netdev_queue *txq;
69 struct sk_buff *skb;
70
71 txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
72 skb = skb_peek(&txp->tq);
73 if (!skb) {
74 if (!__netif_tx_trylock(txq))
75 goto resched;
76 skb_queue_splice_tail_init(&txp->rq, &txp->tq);
77 __netif_tx_unlock(txq);
78 }
79
80 while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
81 u32 from = G_TC_FROM(skb->tc_verd);
82
83 skb_reset_tc(skb);
84 skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
85
86 u64_stats_update_begin(&txp->tsync);
87 txp->tx_packets++;
88 txp->tx_bytes += skb->len;
89 u64_stats_update_end(&txp->tsync);
90
91 rcu_read_lock();
92 skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
93 if (!skb->dev) {
94 rcu_read_unlock();
95 dev_kfree_skb(skb);
96 txp->dev->stats.tx_dropped++;
97 if (skb_queue_len(&txp->tq) != 0)
98 goto resched;
99 break;
100 }
101 rcu_read_unlock();
102 skb->skb_iif = txp->dev->ifindex;
103
104 if (from & AT_EGRESS) {
105 dev_queue_xmit(skb);
106 } else if (from & AT_INGRESS) {
107 skb_pull_rcsum(skb, skb->mac_len);
108 netif_receive_skb(skb);
109 } else
110 BUG();
111 }
112
113 if (__netif_tx_trylock(txq)) {
114 skb = skb_peek(&txp->rq);
115 if (!skb) {
116 txp->tasklet_pending = 0;
117 if (netif_tx_queue_stopped(txq))
118 netif_tx_wake_queue(txq);
119 } else {
120 __netif_tx_unlock(txq);
121 goto resched;
122 }
123 __netif_tx_unlock(txq);
124 } else {
125resched:
126 txp->tasklet_pending = 1;
127 tasklet_schedule(&txp->ifb_tasklet);
128 }
129
130}
131
132static void ifb_stats64(struct net_device *dev,
133 struct rtnl_link_stats64 *stats)
134{
135 struct ifb_dev_private *dp = netdev_priv(dev);
136 struct ifb_q_private *txp = dp->tx_private;
137 unsigned int start;
138 u64 packets, bytes;
139 int i;
140
141 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
142 do {
143 start = u64_stats_fetch_begin_irq(&txp->rsync);
144 packets = txp->rx_packets;
145 bytes = txp->rx_bytes;
146 } while (u64_stats_fetch_retry_irq(&txp->rsync, start));
147 stats->rx_packets += packets;
148 stats->rx_bytes += bytes;
149
150 do {
151 start = u64_stats_fetch_begin_irq(&txp->tsync);
152 packets = txp->tx_packets;
153 bytes = txp->tx_bytes;
154 } while (u64_stats_fetch_retry_irq(&txp->tsync, start));
155 stats->tx_packets += packets;
156 stats->tx_bytes += bytes;
157 }
158 stats->rx_dropped = dev->stats.rx_dropped;
159 stats->tx_dropped = dev->stats.tx_dropped;
160}
161
162static int ifb_dev_init(struct net_device *dev)
163{
164 struct ifb_dev_private *dp = netdev_priv(dev);
165 struct ifb_q_private *txp;
166 int i;
167
168 txp = kcalloc(dev->num_tx_queues, sizeof(*txp), GFP_KERNEL);
169 if (!txp)
170 return -ENOMEM;
171 dp->tx_private = txp;
172 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
173 txp->txqnum = i;
174 txp->dev = dev;
175 __skb_queue_head_init(&txp->rq);
176 __skb_queue_head_init(&txp->tq);
177 u64_stats_init(&txp->rsync);
178 u64_stats_init(&txp->tsync);
179 tasklet_init(&txp->ifb_tasklet, ifb_ri_tasklet,
180 (unsigned long)txp);
181 netif_tx_start_queue(netdev_get_tx_queue(dev, i));
182 }
183 return 0;
184}
185
186static const struct net_device_ops ifb_netdev_ops = {
187 .ndo_open = ifb_open,
188 .ndo_stop = ifb_close,
189 .ndo_get_stats64 = ifb_stats64,
190 .ndo_start_xmit = ifb_xmit,
191 .ndo_validate_addr = eth_validate_addr,
192 .ndo_init = ifb_dev_init,
193};
194
195#define IFB_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_FRAGLIST | \
196 NETIF_F_TSO_ECN | NETIF_F_TSO | NETIF_F_TSO6 | \
197 NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX | \
198 NETIF_F_HW_VLAN_STAG_TX)
199
200static void ifb_dev_free(struct net_device *dev)
201{
202 struct ifb_dev_private *dp = netdev_priv(dev);
203 struct ifb_q_private *txp = dp->tx_private;
204 int i;
205
206 for (i = 0; i < dev->num_tx_queues; i++,txp++) {
207 tasklet_kill(&txp->ifb_tasklet);
208 __skb_queue_purge(&txp->rq);
209 __skb_queue_purge(&txp->tq);
210 }
211 kfree(dp->tx_private);
212}
213
214static void ifb_setup(struct net_device *dev)
215{
216
217 dev->netdev_ops = &ifb_netdev_ops;
218
219
220 ether_setup(dev);
221 dev->tx_queue_len = TX_Q_LIMIT;
222
223 dev->features |= IFB_FEATURES;
224 dev->vlan_features |= IFB_FEATURES;
225
226 dev->flags |= IFF_NOARP;
227 dev->flags &= ~IFF_MULTICAST;
228 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
229 netif_keep_dst(dev);
230 eth_hw_addr_random(dev);
231 dev->extended->needs_free_netdev = true;
232 dev->extended->priv_destructor = ifb_dev_free;
233
234 dev->extended->min_mtu = 0;
235 dev->extended->max_mtu = 0;
236}
237
238static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
239{
240 struct ifb_dev_private *dp = netdev_priv(dev);
241 struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);
242
243 u64_stats_update_begin(&txp->rsync);
244 txp->rx_packets++;
245 txp->rx_bytes += skb->len;
246 u64_stats_update_end(&txp->rsync);
247
248 if (G_TC_FROM(skb->tc_verd) == AT_STACK || !skb->skb_iif) {
249 dev_kfree_skb(skb);
250 dev->stats.rx_dropped++;
251 return NETDEV_TX_OK;
252 }
253
254 if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
255 netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));
256
257 __skb_queue_tail(&txp->rq, skb);
258 if (!txp->tasklet_pending) {
259 txp->tasklet_pending = 1;
260 tasklet_schedule(&txp->ifb_tasklet);
261 }
262
263 return NETDEV_TX_OK;
264}
265
266static int ifb_close(struct net_device *dev)
267{
268 netif_tx_stop_all_queues(dev);
269 return 0;
270}
271
272static int ifb_open(struct net_device *dev)
273{
274 netif_tx_start_all_queues(dev);
275 return 0;
276}
277
278static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
279{
280 if (tb[IFLA_ADDRESS]) {
281 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
282 return -EINVAL;
283 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
284 return -EADDRNOTAVAIL;
285 }
286 return 0;
287}
288
289static struct rtnl_link_ops ifb_link_ops __read_mostly = {
290 .kind = "ifb",
291 .priv_size = sizeof(struct ifb_dev_private),
292 .setup = ifb_setup,
293 .validate = ifb_validate,
294};
295
296
297
298
299
300static int numifbs = 2;
301module_param(numifbs, int, 0);
302MODULE_PARM_DESC(numifbs, "Number of ifb devices");
303
304static int __init ifb_init_one(int index)
305{
306 struct net_device *dev_ifb;
307 int err;
308
309 dev_ifb = alloc_netdev(sizeof(struct ifb_dev_private),
310 "ifb%d", ifb_setup);
311
312 if (!dev_ifb)
313 return -ENOMEM;
314
315 dev_ifb->rtnl_link_ops = &ifb_link_ops;
316 err = register_netdevice(dev_ifb);
317 if (err < 0)
318 goto err;
319
320 return 0;
321
322err:
323 free_netdev(dev_ifb);
324 return err;
325}
326
327static int __init ifb_init_module(void)
328{
329 int i, err;
330
331 rtnl_lock();
332 err = __rtnl_link_register(&ifb_link_ops);
333 if (err < 0)
334 goto out;
335
336 for (i = 0; i < numifbs && !err; i++) {
337 err = ifb_init_one(i);
338 cond_resched();
339 }
340 if (err)
341 __rtnl_link_unregister(&ifb_link_ops);
342
343out:
344 rtnl_unlock();
345
346 return err;
347}
348
349static void __exit ifb_cleanup_module(void)
350{
351 rtnl_link_unregister(&ifb_link_ops);
352}
353
354module_init(ifb_init_module);
355module_exit(ifb_cleanup_module);
356MODULE_LICENSE("GPL");
357MODULE_AUTHOR("Jamal Hadi Salim");
358MODULE_ALIAS_RTNL_LINK("ifb");
359
