1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/etherdevice.h>
35#include <linux/tcp.h>
36#include <linux/if_vlan.h>
37#include <linux/delay.h>
38#include <linux/slab.h>
39#include <linux/hash.h>
40#include <net/ip.h>
41#include <net/busy_poll.h>
42
43#include <linux/mlx4/driver.h>
44#include <linux/mlx4/device.h>
45#include <linux/mlx4/cmd.h>
46#include <linux/mlx4/cq.h>
47
48#include "mlx4_en.h"
49#include "en_port.h"
50
51int mlx4_en_setup_tc(struct net_device *dev, u8 up)
52{
53 struct mlx4_en_priv *priv = netdev_priv(dev);
54 int i;
55 unsigned int offset = 0;
56
57 if (up && up != MLX4_EN_NUM_UP)
58 return -EINVAL;
59
60 netdev_set_num_tc(dev, up);
61
62
63 for (i = 0; i < up; i++) {
64 netdev_set_tc_queue(dev, i, priv->num_tx_rings_p_up, offset);
65 offset += priv->num_tx_rings_p_up;
66 }
67
68 return 0;
69}
70
71#ifdef CONFIG_NET_RX_BUSY_POLL
72
73static int mlx4_en_low_latency_recv(struct napi_struct *napi)
74{
75 struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
76 struct net_device *dev = cq->dev;
77 struct mlx4_en_priv *priv = netdev_priv(dev);
78 struct mlx4_en_rx_ring *rx_ring = &priv->rx_ring[cq->ring];
79 int done;
80
81 if (!priv->port_up)
82 return LL_FLUSH_FAILED;
83
84 if (!mlx4_en_cq_lock_poll(cq))
85 return LL_FLUSH_BUSY;
86
87 done = mlx4_en_process_rx_cq(dev, cq, 4);
88 if (likely(done))
89 rx_ring->cleaned += done;
90 else
91 rx_ring->misses++;
92
93 mlx4_en_cq_unlock_poll(cq);
94
95 return done;
96}
97#endif
98
99#ifdef CONFIG_RFS_ACCEL
100
101struct mlx4_en_filter {
102 struct list_head next;
103 struct work_struct work;
104
105 __be32 src_ip;
106 __be32 dst_ip;
107 __be16 src_port;
108 __be16 dst_port;
109
110 int rxq_index;
111 struct mlx4_en_priv *priv;
112 u32 flow_id;
113 int id;
114 u64 reg_id;
115 u8 activated;
116
117
118 struct hlist_node filter_chain;
119};
120
121static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv);
122
123static void mlx4_en_filter_work(struct work_struct *work)
124{
125 struct mlx4_en_filter *filter = container_of(work,
126 struct mlx4_en_filter,
127 work);
128 struct mlx4_en_priv *priv = filter->priv;
129 struct mlx4_spec_list spec_tcp = {
130 .id = MLX4_NET_TRANS_RULE_ID_TCP,
131 {
132 .tcp_udp = {
133 .dst_port = filter->dst_port,
134 .dst_port_msk = (__force __be16)-1,
135 .src_port = filter->src_port,
136 .src_port_msk = (__force __be16)-1,
137 },
138 },
139 };
140 struct mlx4_spec_list spec_ip = {
141 .id = MLX4_NET_TRANS_RULE_ID_IPV4,
142 {
143 .ipv4 = {
144 .dst_ip = filter->dst_ip,
145 .dst_ip_msk = (__force __be32)-1,
146 .src_ip = filter->src_ip,
147 .src_ip_msk = (__force __be32)-1,
148 },
149 },
150 };
151 struct mlx4_spec_list spec_eth = {
152 .id = MLX4_NET_TRANS_RULE_ID_ETH,
153 };
154 struct mlx4_net_trans_rule rule = {
155 .list = LIST_HEAD_INIT(rule.list),
156 .queue_mode = MLX4_NET_TRANS_Q_LIFO,
157 .exclusive = 1,
158 .allow_loopback = 1,
159 .promisc_mode = MLX4_FS_REGULAR,
160 .port = priv->port,
161 .priority = MLX4_DOMAIN_RFS,
162 };
163 int rc;
164 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
165
166 list_add_tail(&spec_eth.list, &rule.list);
167 list_add_tail(&spec_ip.list, &rule.list);
168 list_add_tail(&spec_tcp.list, &rule.list);
169
170 rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn;
171 memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN);
172 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
173
174 filter->activated = 0;
175
176 if (filter->reg_id) {
177 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id);
178 if (rc && rc != -ENOENT)
179 en_err(priv, "Error detaching flow. rc = %d\n", rc);
180 }
181
182 rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id);
183 if (rc)
184 en_err(priv, "Error attaching flow. err = %d\n", rc);
185
186 mlx4_en_filter_rfs_expire(priv);
187
188 filter->activated = 1;
189}
190
191static inline struct hlist_head *
192filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip,
193 __be16 src_port, __be16 dst_port)
194{
195 unsigned long l;
196 int bucket_idx;
197
198 l = (__force unsigned long)src_port |
199 ((__force unsigned long)dst_port << 2);
200 l ^= (__force unsigned long)(src_ip ^ dst_ip);
201
202 bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT);
203
204 return &priv->filter_hash[bucket_idx];
205}
206
207static struct mlx4_en_filter *
208mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip,
209 __be32 dst_ip, __be16 src_port, __be16 dst_port,
210 u32 flow_id)
211{
212 struct mlx4_en_filter *filter = NULL;
213
214 filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC);
215 if (!filter)
216 return NULL;
217
218 filter->priv = priv;
219 filter->rxq_index = rxq_index;
220 INIT_WORK(&filter->work, mlx4_en_filter_work);
221
222 filter->src_ip = src_ip;
223 filter->dst_ip = dst_ip;
224 filter->src_port = src_port;
225 filter->dst_port = dst_port;
226
227 filter->flow_id = flow_id;
228
229 filter->id = priv->last_filter_id++ % RPS_NO_FILTER;
230
231 list_add_tail(&filter->next, &priv->filters);
232 hlist_add_head(&filter->filter_chain,
233 filter_hash_bucket(priv, src_ip, dst_ip, src_port,
234 dst_port));
235
236 return filter;
237}
238
239static void mlx4_en_filter_free(struct mlx4_en_filter *filter)
240{
241 struct mlx4_en_priv *priv = filter->priv;
242 int rc;
243
244 list_del(&filter->next);
245
246 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id);
247 if (rc && rc != -ENOENT)
248 en_err(priv, "Error detaching flow. rc = %d\n", rc);
249
250 kfree(filter);
251}
252
253static inline struct mlx4_en_filter *
254mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip,
255 __be16 src_port, __be16 dst_port)
256{
257 struct mlx4_en_filter *filter;
258 struct mlx4_en_filter *ret = NULL;
259
260 hlist_for_each_entry(filter,
261 filter_hash_bucket(priv, src_ip, dst_ip,
262 src_port, dst_port),
263 filter_chain) {
264 if (filter->src_ip == src_ip &&
265 filter->dst_ip == dst_ip &&
266 filter->src_port == src_port &&
267 filter->dst_port == dst_port) {
268 ret = filter;
269 break;
270 }
271 }
272
273 return ret;
274}
275
276static int
277mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
278 u16 rxq_index, u32 flow_id)
279{
280 struct mlx4_en_priv *priv = netdev_priv(net_dev);
281 struct mlx4_en_filter *filter;
282 const struct iphdr *ip;
283 const __be16 *ports;
284 __be32 src_ip;
285 __be32 dst_ip;
286 __be16 src_port;
287 __be16 dst_port;
288 int nhoff = skb_network_offset(skb);
289 int ret = 0;
290
291 if (skb->protocol != htons(ETH_P_IP))
292 return -EPROTONOSUPPORT;
293
294 ip = (const struct iphdr *)(skb->data + nhoff);
295 if (ip_is_fragment(ip))
296 return -EPROTONOSUPPORT;
297
298 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
299
300 src_ip = ip->saddr;
301 dst_ip = ip->daddr;
302 src_port = ports[0];
303 dst_port = ports[1];
304
305 if (ip->protocol != IPPROTO_TCP)
306 return -EPROTONOSUPPORT;
307
308 spin_lock_bh(&priv->filters_lock);
309 filter = mlx4_en_filter_find(priv, src_ip, dst_ip, src_port, dst_port);
310 if (filter) {
311 if (filter->rxq_index == rxq_index)
312 goto out;
313
314 filter->rxq_index = rxq_index;
315 } else {
316 filter = mlx4_en_filter_alloc(priv, rxq_index,
317 src_ip, dst_ip,
318 src_port, dst_port, flow_id);
319 if (!filter) {
320 ret = -ENOMEM;
321 goto err;
322 }
323 }
324
325 queue_work(priv->mdev->workqueue, &filter->work);
326
327out:
328 ret = filter->id;
329err:
330 spin_unlock_bh(&priv->filters_lock);
331
332 return ret;
333}
334
335void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv,
336 struct mlx4_en_rx_ring *rx_ring)
337{
338 struct mlx4_en_filter *filter, *tmp;
339 LIST_HEAD(del_list);
340
341 spin_lock_bh(&priv->filters_lock);
342 list_for_each_entry_safe(filter, tmp, &priv->filters, next) {
343 list_move(&filter->next, &del_list);
344 hlist_del(&filter->filter_chain);
345 }
346 spin_unlock_bh(&priv->filters_lock);
347
348 list_for_each_entry_safe(filter, tmp, &del_list, next) {
349 cancel_work_sync(&filter->work);
350 mlx4_en_filter_free(filter);
351 }
352}
353
354static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv)
355{
356 struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL;
357 LIST_HEAD(del_list);
358 int i = 0;
359
360 spin_lock_bh(&priv->filters_lock);
361 list_for_each_entry_safe(filter, tmp, &priv->filters, next) {
362 if (i > MLX4_EN_FILTER_EXPIRY_QUOTA)
363 break;
364
365 if (filter->activated &&
366 !work_pending(&filter->work) &&
367 rps_may_expire_flow(priv->dev,
368 filter->rxq_index, filter->flow_id,
369 filter->id)) {
370 list_move(&filter->next, &del_list);
371 hlist_del(&filter->filter_chain);
372 } else
373 last_filter = filter;
374
375 i++;
376 }
377
378 if (last_filter && (&last_filter->next != priv->filters.next))
379 list_move(&priv->filters, &last_filter->next);
380
381 spin_unlock_bh(&priv->filters_lock);
382
383 list_for_each_entry_safe(filter, tmp, &del_list, next)
384 mlx4_en_filter_free(filter);
385}
386#endif
387
388static int mlx4_en_vlan_rx_add_vid(struct net_device *dev,
389 __be16 proto, u16 vid)
390{
391 struct mlx4_en_priv *priv = netdev_priv(dev);
392 struct mlx4_en_dev *mdev = priv->mdev;
393 int err;
394 int idx;
395
396 en_dbg(HW, priv, "adding VLAN:%d\n", vid);
397
398 set_bit(vid, priv->active_vlans);
399
400
401 mutex_lock(&mdev->state_lock);
402 if (mdev->device_up && priv->port_up) {
403 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
404 if (err)
405 en_err(priv, "Failed configuring VLAN filter\n");
406 }
407 if (mlx4_register_vlan(mdev->dev, priv->port, vid, &idx))
408 en_dbg(HW, priv, "failed adding vlan %d\n", vid);
409 mutex_unlock(&mdev->state_lock);
410
411 return 0;
412}
413
414static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev,
415 __be16 proto, u16 vid)
416{
417 struct mlx4_en_priv *priv = netdev_priv(dev);
418 struct mlx4_en_dev *mdev = priv->mdev;
419 int err;
420 int idx;
421
422 en_dbg(HW, priv, "Killing VID:%d\n", vid);
423
424 clear_bit(vid, priv->active_vlans);
425
426
427 mutex_lock(&mdev->state_lock);
428 if (!mlx4_find_cached_vlan(mdev->dev, priv->port, vid, &idx))
429 mlx4_unregister_vlan(mdev->dev, priv->port, idx);
430 else
431 en_dbg(HW, priv, "could not find vid %d in cache\n", vid);
432
433 if (mdev->device_up && priv->port_up) {
434 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
435 if (err)
436 en_err(priv, "Failed configuring VLAN filter\n");
437 }
438 mutex_unlock(&mdev->state_lock);
439
440 return 0;
441}
442
443static void mlx4_en_u64_to_mac(unsigned char dst_mac[ETH_ALEN + 2], u64 src_mac)
444{
445 int i;
446 for (i = ETH_ALEN - 1; i >= 0; --i) {
447 dst_mac[i] = src_mac & 0xff;
448 src_mac >>= 8;
449 }
450 memset(&dst_mac[ETH_ALEN], 0, 2);
451}
452
453static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv,
454 unsigned char *mac, int *qpn, u64 *reg_id)
455{
456 struct mlx4_en_dev *mdev = priv->mdev;
457 struct mlx4_dev *dev = mdev->dev;
458 int err;
459
460 switch (dev->caps.steering_mode) {
461 case MLX4_STEERING_MODE_B0: {
462 struct mlx4_qp qp;
463 u8 gid[16] = {0};
464
465 qp.qpn = *qpn;
466 memcpy(&gid[10], mac, ETH_ALEN);
467 gid[5] = priv->port;
468
469 err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH);
470 break;
471 }
472 case MLX4_STEERING_MODE_DEVICE_MANAGED: {
473 struct mlx4_spec_list spec_eth = { {NULL} };
474 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
475
476 struct mlx4_net_trans_rule rule = {
477 .queue_mode = MLX4_NET_TRANS_Q_FIFO,
478 .exclusive = 0,
479 .allow_loopback = 1,
480 .promisc_mode = MLX4_FS_REGULAR,
481 .priority = MLX4_DOMAIN_NIC,
482 };
483
484 rule.port = priv->port;
485 rule.qpn = *qpn;
486 INIT_LIST_HEAD(&rule.list);
487
488 spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH;
489 memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN);
490 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
491 list_add_tail(&spec_eth.list, &rule.list);
492
493 err = mlx4_flow_attach(dev, &rule, reg_id);
494 break;
495 }
496 default:
497 return -EINVAL;
498 }
499 if (err)
500 en_warn(priv, "Failed Attaching Unicast\n");
501
502 return err;
503}
504
505static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv,
506 unsigned char *mac, int qpn, u64 reg_id)
507{
508 struct mlx4_en_dev *mdev = priv->mdev;
509 struct mlx4_dev *dev = mdev->dev;
510
511 switch (dev->caps.steering_mode) {
512 case MLX4_STEERING_MODE_B0: {
513 struct mlx4_qp qp;
514 u8 gid[16] = {0};
515
516 qp.qpn = qpn;
517 memcpy(&gid[10], mac, ETH_ALEN);
518 gid[5] = priv->port;
519
520 mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH);
521 break;
522 }
523 case MLX4_STEERING_MODE_DEVICE_MANAGED: {
524 mlx4_flow_detach(dev, reg_id);
525 break;
526 }
527 default:
528 en_err(priv, "Invalid steering mode.\n");
529 }
530}
531
532static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
533{
534 struct mlx4_en_dev *mdev = priv->mdev;
535 struct mlx4_dev *dev = mdev->dev;
536 struct mlx4_mac_entry *entry;
537 int index = 0;
538 int err = 0;
539 u64 reg_id;
540 int *qpn = &priv->base_qpn;
541 u64 mac = mlx4_en_mac_to_u64(priv->dev->dev_addr);
542
543 en_dbg(DRV, priv, "Registering MAC: %pM for adding\n",
544 priv->dev->dev_addr);
545 index = mlx4_register_mac(dev, priv->port, mac);
546 if (index < 0) {
547 err = index;
548 en_err(priv, "Failed adding MAC: %pM\n",
549 priv->dev->dev_addr);
550 return err;
551 }
552
553 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) {
554 int base_qpn = mlx4_get_base_qpn(dev, priv->port);
555 *qpn = base_qpn + index;
556 return 0;
557 }
558
559 err = mlx4_qp_reserve_range(dev, 1, 1, qpn);
560 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
561 if (err) {
562 en_err(priv, "Failed to reserve qp for mac registration\n");
563 goto qp_err;
564 }
565
566 err = mlx4_en_uc_steer_add(priv, priv->dev->dev_addr, qpn, ®_id);
567 if (err)
568 goto steer_err;
569
570 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
571 if (!entry) {
572 err = -ENOMEM;
573 goto alloc_err;
574 }
575 memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac));
576 entry->reg_id = reg_id;
577
578 hlist_add_head_rcu(&entry->hlist,
579 &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]);
580
581 return 0;
582
583alloc_err:
584 mlx4_en_uc_steer_release(priv, priv->dev->dev_addr, *qpn, reg_id);
585
586steer_err:
587 mlx4_qp_release_range(dev, *qpn, 1);
588
589qp_err:
590 mlx4_unregister_mac(dev, priv->port, mac);
591 return err;
592}
593
594static void mlx4_en_put_qp(struct mlx4_en_priv *priv)
595{
596 struct mlx4_en_dev *mdev = priv->mdev;
597 struct mlx4_dev *dev = mdev->dev;
598 int qpn = priv->base_qpn;
599 u64 mac;
600
601 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) {
602 mac = mlx4_en_mac_to_u64(priv->dev->dev_addr);
603 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n",
604 priv->dev->dev_addr);
605 mlx4_unregister_mac(dev, priv->port, mac);
606 } else {
607 struct mlx4_mac_entry *entry;
608 struct hlist_node *tmp;
609 struct hlist_head *bucket;
610 unsigned int i;
611
612 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) {
613 bucket = &priv->mac_hash[i];
614 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
615 mac = mlx4_en_mac_to_u64(entry->mac);
616 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n",
617 entry->mac);
618 mlx4_en_uc_steer_release(priv, entry->mac,
619 qpn, entry->reg_id);
620
621 mlx4_unregister_mac(dev, priv->port, mac);
622 hlist_del_rcu(&entry->hlist);
623 kfree_rcu(entry, rcu);
624 }
625 }
626
627 en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n",
628 priv->port, qpn);
629 mlx4_qp_release_range(dev, qpn, 1);
630 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC;
631 }
632}
633
634static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn,
635 unsigned char *new_mac, unsigned char *prev_mac)
636{
637 struct mlx4_en_dev *mdev = priv->mdev;
638 struct mlx4_dev *dev = mdev->dev;
639 int err = 0;
640 u64 new_mac_u64 = mlx4_en_mac_to_u64(new_mac);
641
642 if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) {
643 struct hlist_head *bucket;
644 unsigned int mac_hash;
645 struct mlx4_mac_entry *entry;
646 struct hlist_node *tmp;
647 u64 prev_mac_u64 = mlx4_en_mac_to_u64(prev_mac);
648
649 bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]];
650 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
651 if (ether_addr_equal_64bits(entry->mac, prev_mac)) {
652 mlx4_en_uc_steer_release(priv, entry->mac,
653 qpn, entry->reg_id);
654 mlx4_unregister_mac(dev, priv->port,
655 prev_mac_u64);
656 hlist_del_rcu(&entry->hlist);
657 synchronize_rcu();
658 memcpy(entry->mac, new_mac, ETH_ALEN);
659 entry->reg_id = 0;
660 mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX];
661 hlist_add_head_rcu(&entry->hlist,
662 &priv->mac_hash[mac_hash]);
663 mlx4_register_mac(dev, priv->port, new_mac_u64);
664 err = mlx4_en_uc_steer_add(priv, new_mac,
665 &qpn,
666 &entry->reg_id);
667 return err;
668 }
669 }
670 return -EINVAL;
671 }
672
673 return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64);
674}
675
676u64 mlx4_en_mac_to_u64(u8 *addr)
677{
678 u64 mac = 0;
679 int i;
680
681 for (i = 0; i < ETH_ALEN; i++) {
682 mac <<= 8;
683 mac |= addr[i];
684 }
685 return mac;
686}
687
688static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv)
689{
690 int err = 0;
691
692 if (priv->port_up) {
693
694 err = mlx4_en_replace_mac(priv, priv->base_qpn,
695 priv->dev->dev_addr, priv->prev_mac);
696 if (err)
697 en_err(priv, "Failed changing HW MAC address\n");
698 memcpy(priv->prev_mac, priv->dev->dev_addr,
699 sizeof(priv->prev_mac));
700 } else
701 en_dbg(HW, priv, "Port is down while registering mac, exiting...\n");
702
703 return err;
704}
705
706static int mlx4_en_set_mac(struct net_device *dev, void *addr)
707{
708 struct mlx4_en_priv *priv = netdev_priv(dev);
709 struct mlx4_en_dev *mdev = priv->mdev;
710 struct sockaddr *saddr = addr;
711 int err;
712
713 if (!is_valid_ether_addr(saddr->sa_data))
714 return -EADDRNOTAVAIL;
715
716 memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN);
717
718 mutex_lock(&mdev->state_lock);
719 err = mlx4_en_do_set_mac(priv);
720 mutex_unlock(&mdev->state_lock);
721
722 return err;
723}
724
725static void mlx4_en_clear_list(struct net_device *dev)
726{
727 struct mlx4_en_priv *priv = netdev_priv(dev);
728 struct mlx4_en_mc_list *tmp, *mc_to_del;
729
730 list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) {
731 list_del(&mc_to_del->list);
732 kfree(mc_to_del);
733 }
734}
735
736static void mlx4_en_cache_mclist(struct net_device *dev)
737{
738 struct mlx4_en_priv *priv = netdev_priv(dev);
739 struct netdev_hw_addr *ha;
740 struct mlx4_en_mc_list *tmp;
741
742 mlx4_en_clear_list(dev);
743 netdev_for_each_mc_addr(ha, dev) {
744 tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC);
745 if (!tmp) {
746 mlx4_en_clear_list(dev);
747 return;
748 }
749 memcpy(tmp->addr, ha->addr, ETH_ALEN);
750 list_add_tail(&tmp->list, &priv->mc_list);
751 }
752}
753
754static void update_mclist_flags(struct mlx4_en_priv *priv,
755 struct list_head *dst,
756 struct list_head *src)
757{
758 struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc;
759 bool found;
760
761
762
763
764 list_for_each_entry(dst_tmp, dst, list) {
765 found = false;
766 list_for_each_entry(src_tmp, src, list) {
767 if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) {
768 found = true;
769 break;
770 }
771 }
772 if (!found)
773 dst_tmp->action = MCLIST_REM;
774 }
775
776
777
778
779 list_for_each_entry(src_tmp, src, list) {
780 found = false;
781 list_for_each_entry(dst_tmp, dst, list) {
782 if (!memcmp(dst_tmp->addr, src_tmp->addr, ETH_ALEN)) {
783 dst_tmp->action = MCLIST_NONE;
784 found = true;
785 break;
786 }
787 }
788 if (!found) {
789 new_mc = kmemdup(src_tmp,
790 sizeof(struct mlx4_en_mc_list),
791 GFP_KERNEL);
792 if (!new_mc)
793 return;
794
795 new_mc->action = MCLIST_ADD;
796 list_add_tail(&new_mc->list, dst);
797 }
798 }
799}
800
801static void mlx4_en_set_rx_mode(struct net_device *dev)
802{
803 struct mlx4_en_priv *priv = netdev_priv(dev);
804
805 if (!priv->port_up)
806 return;
807
808 queue_work(priv->mdev->workqueue, &priv->rx_mode_task);
809}
810
811static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv,
812 struct mlx4_en_dev *mdev)
813{
814 int err = 0;
815
816 if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) {
817 if (netif_msg_rx_status(priv))
818 en_warn(priv, "Entering promiscuous mode\n");
819 priv->flags |= MLX4_EN_FLAG_PROMISC;
820
821
822 switch (mdev->dev->caps.steering_mode) {
823 case MLX4_STEERING_MODE_DEVICE_MANAGED:
824 err = mlx4_flow_steer_promisc_add(mdev->dev,
825 priv->port,
826 priv->base_qpn,
827 MLX4_FS_ALL_DEFAULT);
828 if (err)
829 en_err(priv, "Failed enabling promiscuous mode\n");
830 priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
831 break;
832
833 case MLX4_STEERING_MODE_B0:
834 err = mlx4_unicast_promisc_add(mdev->dev,
835 priv->base_qpn,
836 priv->port);
837 if (err)
838 en_err(priv, "Failed enabling unicast promiscuous mode\n");
839
840
841
842
843 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) {
844 err = mlx4_multicast_promisc_add(mdev->dev,
845 priv->base_qpn,
846 priv->port);
847 if (err)
848 en_err(priv, "Failed enabling multicast promiscuous mode\n");
849 priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
850 }
851 break;
852
853 case MLX4_STEERING_MODE_A0:
854 err = mlx4_SET_PORT_qpn_calc(mdev->dev,
855 priv->port,
856 priv->base_qpn,
857 1);
858 if (err)
859 en_err(priv, "Failed enabling promiscuous mode\n");
860 break;
861 }
862
863
864 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
865 0, MLX4_MCAST_DISABLE);
866 if (err)
867 en_err(priv, "Failed disabling multicast filter\n");
868
869
870 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
871 if (err)
872 en_err(priv, "Failed disabling VLAN filter\n");
873 }
874}
875
876static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv,
877 struct mlx4_en_dev *mdev)
878{
879 int err = 0;
880
881 if (netif_msg_rx_status(priv))
882 en_warn(priv, "Leaving promiscuous mode\n");
883 priv->flags &= ~MLX4_EN_FLAG_PROMISC;
884
885
886 switch (mdev->dev->caps.steering_mode) {
887 case MLX4_STEERING_MODE_DEVICE_MANAGED:
888 err = mlx4_flow_steer_promisc_remove(mdev->dev,
889 priv->port,
890 MLX4_FS_ALL_DEFAULT);
891 if (err)
892 en_err(priv, "Failed disabling promiscuous mode\n");
893 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
894 break;
895
896 case MLX4_STEERING_MODE_B0:
897 err = mlx4_unicast_promisc_remove(mdev->dev,
898 priv->base_qpn,
899 priv->port);
900 if (err)
901 en_err(priv, "Failed disabling unicast promiscuous mode\n");
902
903 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
904 err = mlx4_multicast_promisc_remove(mdev->dev,
905 priv->base_qpn,
906 priv->port);
907 if (err)
908 en_err(priv, "Failed disabling multicast promiscuous mode\n");
909 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
910 }
911 break;
912
913 case MLX4_STEERING_MODE_A0:
914 err = mlx4_SET_PORT_qpn_calc(mdev->dev,
915 priv->port,
916 priv->base_qpn, 0);
917 if (err)
918 en_err(priv, "Failed disabling promiscuous mode\n");
919 break;
920 }
921
922
923 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
924 if (err)
925 en_err(priv, "Failed enabling VLAN filter\n");
926}
927
928static void mlx4_en_do_multicast(struct mlx4_en_priv *priv,
929 struct net_device *dev,
930 struct mlx4_en_dev *mdev)
931{
932 struct mlx4_en_mc_list *mclist, *tmp;
933 u64 mcast_addr = 0;
934 u8 mc_list[16] = {0};
935 int err = 0;
936
937
938 if (dev->flags & IFF_ALLMULTI) {
939 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
940 0, MLX4_MCAST_DISABLE);
941 if (err)
942 en_err(priv, "Failed disabling multicast filter\n");
943
944
945 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) {
946 switch (mdev->dev->caps.steering_mode) {
947 case MLX4_STEERING_MODE_DEVICE_MANAGED:
948 err = mlx4_flow_steer_promisc_add(mdev->dev,
949 priv->port,
950 priv->base_qpn,
951 MLX4_FS_MC_DEFAULT);
952 break;
953
954 case MLX4_STEERING_MODE_B0:
955 err = mlx4_multicast_promisc_add(mdev->dev,
956 priv->base_qpn,
957 priv->port);
958 break;
959
960 case MLX4_STEERING_MODE_A0:
961 break;
962 }
963 if (err)
964 en_err(priv, "Failed entering multicast promisc mode\n");
965 priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
966 }
967 } else {
968
969 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
970 switch (mdev->dev->caps.steering_mode) {
971 case MLX4_STEERING_MODE_DEVICE_MANAGED:
972 err = mlx4_flow_steer_promisc_remove(mdev->dev,
973 priv->port,
974 MLX4_FS_MC_DEFAULT);
975 break;
976
977 case MLX4_STEERING_MODE_B0:
978 err = mlx4_multicast_promisc_remove(mdev->dev,
979 priv->base_qpn,
980 priv->port);
981 break;
982
983 case MLX4_STEERING_MODE_A0:
984 break;
985 }
986 if (err)
987 en_err(priv, "Failed disabling multicast promiscuous mode\n");
988 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
989 }
990
991 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
992 0, MLX4_MCAST_DISABLE);
993 if (err)
994 en_err(priv, "Failed disabling multicast filter\n");
995
996
997 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST,
998 1, MLX4_MCAST_CONFIG);
999
1000
1001
1002 netif_addr_lock_bh(dev);
1003 mlx4_en_cache_mclist(dev);
1004 netif_addr_unlock_bh(dev);
1005 list_for_each_entry(mclist, &priv->mc_list, list) {
1006 mcast_addr = mlx4_en_mac_to_u64(mclist->addr);
1007 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port,
1008 mcast_addr, 0, MLX4_MCAST_CONFIG);
1009 }
1010 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
1011 0, MLX4_MCAST_ENABLE);
1012 if (err)
1013 en_err(priv, "Failed enabling multicast filter\n");
1014
1015 update_mclist_flags(priv, &priv->curr_list, &priv->mc_list);
1016 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) {
1017 if (mclist->action == MCLIST_REM) {
1018
1019 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
1020 mc_list[5] = priv->port;
1021 err = mlx4_multicast_detach(mdev->dev,
1022 &priv->rss_map.indir_qp,
1023 mc_list,
1024 MLX4_PROT_ETH,
1025 mclist->reg_id);
1026 if (err)
1027 en_err(priv, "Fail to detach multicast address\n");
1028
1029
1030 list_del(&mclist->list);
1031 kfree(mclist);
1032 } else if (mclist->action == MCLIST_ADD) {
1033
1034 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
1035
1036 mc_list[5] = priv->port;
1037 err = mlx4_multicast_attach(mdev->dev,
1038 &priv->rss_map.indir_qp,
1039 mc_list,
1040 priv->port, 0,
1041 MLX4_PROT_ETH,
1042 &mclist->reg_id);
1043 if (err)
1044 en_err(priv, "Fail to attach multicast address\n");
1045
1046 }
1047 }
1048 }
1049}
1050
1051static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv,
1052 struct net_device *dev,
1053 struct mlx4_en_dev *mdev)
1054{
1055 struct netdev_hw_addr *ha;
1056 struct mlx4_mac_entry *entry;
1057 struct hlist_node *tmp;
1058 bool found;
1059 u64 mac;
1060 int err = 0;
1061 struct hlist_head *bucket;
1062 unsigned int i;
1063 int removed = 0;
1064 u32 prev_flags;
1065
1066
1067
1068
1069
1070
1071 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) {
1072 bucket = &priv->mac_hash[i];
1073 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
1074 found = false;
1075 netdev_for_each_uc_addr(ha, dev) {
1076 if (ether_addr_equal_64bits(entry->mac,
1077 ha->addr)) {
1078 found = true;
1079 break;
1080 }
1081 }
1082
1083
1084 if (ether_addr_equal_64bits(entry->mac, dev->dev_addr))
1085 found = true;
1086
1087 if (!found) {
1088 mac = mlx4_en_mac_to_u64(entry->mac);
1089 mlx4_en_uc_steer_release(priv, entry->mac,
1090 priv->base_qpn,
1091 entry->reg_id);
1092 mlx4_unregister_mac(mdev->dev, priv->port, mac);
1093
1094 hlist_del_rcu(&entry->hlist);
1095 kfree_rcu(entry, rcu);
1096 en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n",
1097 entry->mac, priv->port);
1098 ++removed;
1099 }
1100 }
1101 }
1102
1103
1104
1105
1106 if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed)
1107 return;
1108
1109 prev_flags = priv->flags;
1110 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC;
1111
1112
1113 netdev_for_each_uc_addr(ha, dev) {
1114 found = false;
1115 bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]];
1116 hlist_for_each_entry(entry, bucket, hlist) {
1117 if (ether_addr_equal_64bits(entry->mac, ha->addr)) {
1118 found = true;
1119 break;
1120 }
1121 }
1122
1123 if (!found) {
1124 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1125 if (!entry) {
1126 en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n",
1127 ha->addr, priv->port);
1128 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1129 break;
1130 }
1131 mac = mlx4_en_mac_to_u64(ha->addr);
1132 memcpy(entry->mac, ha->addr, ETH_ALEN);
1133 err = mlx4_register_mac(mdev->dev, priv->port, mac);
1134 if (err < 0) {
1135 en_err(priv, "Failed registering MAC %pM on port %d: %d\n",
1136 ha->addr, priv->port, err);
1137 kfree(entry);
1138 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1139 break;
1140 }
1141 err = mlx4_en_uc_steer_add(priv, ha->addr,
1142 &priv->base_qpn,
1143 &entry->reg_id);
1144 if (err) {
1145 en_err(priv, "Failed adding MAC %pM on port %d: %d\n",
1146 ha->addr, priv->port, err);
1147 mlx4_unregister_mac(mdev->dev, priv->port, mac);
1148 kfree(entry);
1149 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1150 break;
1151 } else {
1152 unsigned int mac_hash;
1153 en_dbg(DRV, priv, "Added MAC %pM on port:%d\n",
1154 ha->addr, priv->port);
1155 mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX];
1156 bucket = &priv->mac_hash[mac_hash];
1157 hlist_add_head_rcu(&entry->hlist, bucket);
1158 }
1159 }
1160 }
1161
1162 if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) {
1163 en_warn(priv, "Forcing promiscuous mode on port:%d\n",
1164 priv->port);
1165 } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) {
1166 en_warn(priv, "Stop forcing promiscuous mode on port:%d\n",
1167 priv->port);
1168 }
1169}
1170
1171static void mlx4_en_do_set_rx_mode(struct work_struct *work)
1172{
1173 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1174 rx_mode_task);
1175 struct mlx4_en_dev *mdev = priv->mdev;
1176 struct net_device *dev = priv->dev;
1177
1178 mutex_lock(&mdev->state_lock);
1179 if (!mdev->device_up) {
1180 en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n");
1181 goto out;
1182 }
1183 if (!priv->port_up) {
1184 en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n");
1185 goto out;
1186 }
1187
1188 if (!netif_carrier_ok(dev)) {
1189 if (!mlx4_en_QUERY_PORT(mdev, priv->port)) {
1190 if (priv->port_state.link_state) {
1191 priv->last_link_state = MLX4_DEV_EVENT_PORT_UP;
1192 netif_carrier_on(dev);
1193 en_dbg(LINK, priv, "Link Up\n");
1194 }
1195 }
1196 }
1197
1198 if (dev->priv_flags & IFF_UNICAST_FLT)
1199 mlx4_en_do_uc_filter(priv, dev, mdev);
1200
1201
1202 if ((dev->flags & IFF_PROMISC) ||
1203 (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) {
1204 mlx4_en_set_promisc_mode(priv, mdev);
1205 goto out;
1206 }
1207
1208
1209 if (priv->flags & MLX4_EN_FLAG_PROMISC)
1210 mlx4_en_clear_promisc_mode(priv, mdev);
1211
1212 mlx4_en_do_multicast(priv, dev, mdev);
1213out:
1214 mutex_unlock(&mdev->state_lock);
1215}
1216
1217#ifdef CONFIG_NET_POLL_CONTROLLER
1218static void mlx4_en_netpoll(struct net_device *dev)
1219{
1220 struct mlx4_en_priv *priv = netdev_priv(dev);
1221 struct mlx4_en_cq *cq;
1222 unsigned long flags;
1223 int i;
1224
1225 for (i = 0; i < priv->rx_ring_num; i++) {
1226 cq = &priv->rx_cq[i];
1227 spin_lock_irqsave(&cq->lock, flags);
1228 napi_synchronize(&cq->napi);
1229 mlx4_en_process_rx_cq(dev, cq, 0);
1230 spin_unlock_irqrestore(&cq->lock, flags);
1231 }
1232}
1233#endif
1234
1235static void mlx4_en_tx_timeout(struct net_device *dev)
1236{
1237 struct mlx4_en_priv *priv = netdev_priv(dev);
1238 struct mlx4_en_dev *mdev = priv->mdev;
1239 int i;
1240
1241 if (netif_msg_timer(priv))
1242 en_warn(priv, "Tx timeout called on port:%d\n", priv->port);
1243
1244 for (i = 0; i < priv->tx_ring_num; i++) {
1245 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, i)))
1246 continue;
1247 en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n",
1248 i, priv->tx_ring[i].qpn, priv->tx_ring[i].cqn,
1249 priv->tx_ring[i].cons, priv->tx_ring[i].prod);
1250 }
1251
1252 priv->port_stats.tx_timeout++;
1253 en_dbg(DRV, priv, "Scheduling watchdog\n");
1254 queue_work(mdev->workqueue, &priv->watchdog_task);
1255}
1256
1257
1258static struct net_device_stats *mlx4_en_get_stats(struct net_device *dev)
1259{
1260 struct mlx4_en_priv *priv = netdev_priv(dev);
1261
1262 spin_lock_bh(&priv->stats_lock);
1263 memcpy(&priv->ret_stats, &priv->stats, sizeof(priv->stats));
1264 spin_unlock_bh(&priv->stats_lock);
1265
1266 return &priv->ret_stats;
1267}
1268
1269static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv)
1270{
1271 struct mlx4_en_cq *cq;
1272 int i;
1273
1274
1275
1276
1277
1278
1279
1280 priv->rx_frames = MLX4_EN_RX_COAL_TARGET;
1281 priv->rx_usecs = MLX4_EN_RX_COAL_TIME;
1282 priv->tx_frames = MLX4_EN_TX_COAL_PKTS;
1283 priv->tx_usecs = MLX4_EN_TX_COAL_TIME;
1284 en_dbg(INTR, priv, "Default coalesing params for mtu:%d - rx_frames:%d rx_usecs:%d\n",
1285 priv->dev->mtu, priv->rx_frames, priv->rx_usecs);
1286
1287
1288 for (i = 0; i < priv->rx_ring_num; i++) {
1289 cq = &priv->rx_cq[i];
1290 cq->moder_cnt = priv->rx_frames;
1291 cq->moder_time = priv->rx_usecs;
1292 priv->last_moder_time[i] = MLX4_EN_AUTO_CONF;
1293 priv->last_moder_packets[i] = 0;
1294 priv->last_moder_bytes[i] = 0;
1295 }
1296
1297 for (i = 0; i < priv->tx_ring_num; i++) {
1298 cq = &priv->tx_cq[i];
1299 cq->moder_cnt = priv->tx_frames;
1300 cq->moder_time = priv->tx_usecs;
1301 }
1302
1303
1304 priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW;
1305 priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW;
1306 priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH;
1307 priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH;
1308 priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL;
1309 priv->adaptive_rx_coal = 1;
1310 priv->last_moder_jiffies = 0;
1311 priv->last_moder_tx_packets = 0;
1312}
1313
1314static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv)
1315{
1316 unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies);
1317 struct mlx4_en_cq *cq;
1318 unsigned long packets;
1319 unsigned long rate;
1320 unsigned long avg_pkt_size;
1321 unsigned long rx_packets;
1322 unsigned long rx_bytes;
1323 unsigned long rx_pkt_diff;
1324 int moder_time;
1325 int ring, err;
1326
1327 if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ)
1328 return;
1329
1330 for (ring = 0; ring < priv->rx_ring_num; ring++) {
1331 spin_lock_bh(&priv->stats_lock);
1332 rx_packets = priv->rx_ring[ring].packets;
1333 rx_bytes = priv->rx_ring[ring].bytes;
1334 spin_unlock_bh(&priv->stats_lock);
1335
1336 rx_pkt_diff = ((unsigned long) (rx_packets -
1337 priv->last_moder_packets[ring]));
1338 packets = rx_pkt_diff;
1339 rate = packets * HZ / period;
1340 avg_pkt_size = packets ? ((unsigned long) (rx_bytes -
1341 priv->last_moder_bytes[ring])) / packets : 0;
1342
1343
1344
1345 if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) &&
1346 avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) {
1347 if (rate < priv->pkt_rate_low)
1348 moder_time = priv->rx_usecs_low;
1349 else if (rate > priv->pkt_rate_high)
1350 moder_time = priv->rx_usecs_high;
1351 else
1352 moder_time = (rate - priv->pkt_rate_low) *
1353 (priv->rx_usecs_high - priv->rx_usecs_low) /
1354 (priv->pkt_rate_high - priv->pkt_rate_low) +
1355 priv->rx_usecs_low;
1356 } else {
1357 moder_time = priv->rx_usecs_low;
1358 }
1359
1360 if (moder_time != priv->last_moder_time[ring]) {
1361 priv->last_moder_time[ring] = moder_time;
1362 cq = &priv->rx_cq[ring];
1363 cq->moder_time = moder_time;
1364 cq->moder_cnt = priv->rx_frames;
1365 err = mlx4_en_set_cq_moder(priv, cq);
1366 if (err)
1367 en_err(priv, "Failed modifying moderation for cq:%d\n",
1368 ring);
1369 }
1370 priv->last_moder_packets[ring] = rx_packets;
1371 priv->last_moder_bytes[ring] = rx_bytes;
1372 }
1373
1374 priv->last_moder_jiffies = jiffies;
1375}
1376
1377static void mlx4_en_do_get_stats(struct work_struct *work)
1378{
1379 struct delayed_work *delay = to_delayed_work(work);
1380 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv,
1381 stats_task);
1382 struct mlx4_en_dev *mdev = priv->mdev;
1383 int err;
1384
1385 mutex_lock(&mdev->state_lock);
1386 if (mdev->device_up) {
1387 if (priv->port_up) {
1388 err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0);
1389 if (err)
1390 en_dbg(HW, priv, "Could not update stats\n");
1391
1392 mlx4_en_auto_moderation(priv);
1393 }
1394
1395 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY);
1396 }
1397 if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) {
1398 mlx4_en_do_set_mac(priv);
1399 mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0;
1400 }
1401 mutex_unlock(&mdev->state_lock);
1402}
1403
1404
1405
1406
1407static void mlx4_en_service_task(struct work_struct *work)
1408{
1409 struct delayed_work *delay = to_delayed_work(work);
1410 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv,
1411 service_task);
1412 struct mlx4_en_dev *mdev = priv->mdev;
1413
1414 mutex_lock(&mdev->state_lock);
1415 if (mdev->device_up) {
1416 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)
1417 mlx4_en_ptp_overflow_check(mdev);
1418
1419 queue_delayed_work(mdev->workqueue, &priv->service_task,
1420 SERVICE_TASK_DELAY);
1421 }
1422 mutex_unlock(&mdev->state_lock);
1423}
1424
1425static void mlx4_en_linkstate(struct work_struct *work)
1426{
1427 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1428 linkstate_task);
1429 struct mlx4_en_dev *mdev = priv->mdev;
1430 int linkstate = priv->link_state;
1431
1432 mutex_lock(&mdev->state_lock);
1433
1434
1435 if (priv->last_link_state != linkstate) {
1436 if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) {
1437 en_info(priv, "Link Down\n");
1438 netif_carrier_off(priv->dev);
1439 } else {
1440 en_info(priv, "Link Up\n");
1441 netif_carrier_on(priv->dev);
1442 }
1443 }
1444 priv->last_link_state = linkstate;
1445 mutex_unlock(&mdev->state_lock);
1446}
1447
1448
1449int mlx4_en_start_port(struct net_device *dev)
1450{
1451 struct mlx4_en_priv *priv = netdev_priv(dev);
1452 struct mlx4_en_dev *mdev = priv->mdev;
1453 struct mlx4_en_cq *cq;
1454 struct mlx4_en_tx_ring *tx_ring;
1455 int rx_index = 0;
1456 int tx_index = 0;
1457 int err = 0;
1458 int i;
1459 int j;
1460 u8 mc_list[16] = {0};
1461
1462 if (priv->port_up) {
1463 en_dbg(DRV, priv, "start port called while port already up\n");
1464 return 0;
1465 }
1466
1467 INIT_LIST_HEAD(&priv->mc_list);
1468 INIT_LIST_HEAD(&priv->curr_list);
1469 INIT_LIST_HEAD(&priv->ethtool_list);
1470 memset(&priv->ethtool_rules[0], 0,
1471 sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES);
1472
1473
1474 dev->mtu = min(dev->mtu, priv->max_mtu);
1475 mlx4_en_calc_rx_buf(dev);
1476 en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_skb_size);
1477
1478
1479 err = mlx4_en_activate_rx_rings(priv);
1480 if (err) {
1481 en_err(priv, "Failed to activate RX rings\n");
1482 return err;
1483 }
1484 for (i = 0; i < priv->rx_ring_num; i++) {
1485 cq = &priv->rx_cq[i];
1486
1487 mlx4_en_cq_init_lock(cq);
1488
1489 err = mlx4_en_activate_cq(priv, cq, i);
1490 if (err) {
1491 en_err(priv, "Failed activating Rx CQ\n");
1492 goto cq_err;
1493 }
1494 for (j = 0; j < cq->size; j++)
1495 cq->buf[j].owner_sr_opcode = MLX4_CQE_OWNER_MASK;
1496 err = mlx4_en_set_cq_moder(priv, cq);
1497 if (err) {
1498 en_err(priv, "Failed setting cq moderation parameters");
1499 mlx4_en_deactivate_cq(priv, cq);
1500 goto cq_err;
1501 }
1502 mlx4_en_arm_cq(priv, cq);
1503 priv->rx_ring[i].cqn = cq->mcq.cqn;
1504 ++rx_index;
1505 }
1506
1507
1508 en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port);
1509 err = mlx4_en_get_qp(priv);
1510 if (err) {
1511 en_err(priv, "Failed getting eth qp\n");
1512 goto cq_err;
1513 }
1514 mdev->mac_removed[priv->port] = 0;
1515
1516 err = mlx4_en_config_rss_steer(priv);
1517 if (err) {
1518 en_err(priv, "Failed configuring rss steering\n");
1519 goto mac_err;
1520 }
1521
1522 err = mlx4_en_create_drop_qp(priv);
1523 if (err)
1524 goto rss_err;
1525
1526
1527 for (i = 0; i < priv->tx_ring_num; i++) {
1528
1529 cq = &priv->tx_cq[i];
1530 err = mlx4_en_activate_cq(priv, cq, i);
1531 if (err) {
1532 en_err(priv, "Failed allocating Tx CQ\n");
1533 goto tx_err;
1534 }
1535 err = mlx4_en_set_cq_moder(priv, cq);
1536 if (err) {
1537 en_err(priv, "Failed setting cq moderation parameters");
1538 mlx4_en_deactivate_cq(priv, cq);
1539 goto tx_err;
1540 }
1541 en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i);
1542 cq->buf->wqe_index = cpu_to_be16(0xffff);
1543
1544
1545 tx_ring = &priv->tx_ring[i];
1546 err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
1547 i / priv->num_tx_rings_p_up);
1548 if (err) {
1549 en_err(priv, "Failed allocating Tx ring\n");
1550 mlx4_en_deactivate_cq(priv, cq);
1551 goto tx_err;
1552 }
1553 tx_ring->tx_queue = netdev_get_tx_queue(dev, i);
1554
1555
1556 mlx4_en_arm_cq(priv, cq);
1557
1558
1559 for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
1560 *((u32 *) (tx_ring->buf + j)) = 0xffffffff;
1561 ++tx_index;
1562 }
1563
1564
1565 err = mlx4_SET_PORT_general(mdev->dev, priv->port,
1566 priv->rx_skb_size + ETH_FCS_LEN,
1567 priv->prof->tx_pause,
1568 priv->prof->tx_ppp,
1569 priv->prof->rx_pause,
1570 priv->prof->rx_ppp);
1571 if (err) {
1572 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n",
1573 priv->port, err);
1574 goto tx_err;
1575 }
1576
1577 err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0);
1578 if (err) {
1579 en_err(priv, "Failed setting default qp numbers\n");
1580 goto tx_err;
1581 }
1582
1583
1584 en_dbg(HW, priv, "Initializing port\n");
1585 err = mlx4_INIT_PORT(mdev->dev, priv->port);
1586 if (err) {
1587 en_err(priv, "Failed Initializing port\n");
1588 goto tx_err;
1589 }
1590
1591
1592 memset(&mc_list[10], 0xff, ETH_ALEN);
1593 mc_list[5] = priv->port;
1594 if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list,
1595 priv->port, 0, MLX4_PROT_ETH,
1596 &priv->broadcast_id))
1597 mlx4_warn(mdev, "Failed Attaching Broadcast\n");
1598
1599
1600 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC);
1601
1602
1603 queue_work(mdev->workqueue, &priv->rx_mode_task);
1604
1605 mlx4_set_stats_bitmap(mdev->dev, &priv->stats_bitmap);
1606
1607 priv->port_up = true;
1608 netif_tx_start_all_queues(dev);
1609 netif_device_attach(dev);
1610
1611 return 0;
1612
1613tx_err:
1614 while (tx_index--) {
1615 mlx4_en_deactivate_tx_ring(priv, &priv->tx_ring[tx_index]);
1616 mlx4_en_deactivate_cq(priv, &priv->tx_cq[tx_index]);
1617 }
1618 mlx4_en_destroy_drop_qp(priv);
1619rss_err:
1620 mlx4_en_release_rss_steer(priv);
1621mac_err:
1622 mlx4_en_put_qp(priv);
1623cq_err:
1624 while (rx_index--)
1625 mlx4_en_deactivate_cq(priv, &priv->rx_cq[rx_index]);
1626 for (i = 0; i < priv->rx_ring_num; i++)
1627 mlx4_en_deactivate_rx_ring(priv, &priv->rx_ring[i]);
1628
1629 return err;
1630}
1631
1632
1633void mlx4_en_stop_port(struct net_device *dev, int detach)
1634{
1635 struct mlx4_en_priv *priv = netdev_priv(dev);
1636 struct mlx4_en_dev *mdev = priv->mdev;
1637 struct mlx4_en_mc_list *mclist, *tmp;
1638 struct ethtool_flow_id *flow, *tmp_flow;
1639 int i;
1640 u8 mc_list[16] = {0};
1641
1642 if (!priv->port_up) {
1643 en_dbg(DRV, priv, "stop port called while port already down\n");
1644 return;
1645 }
1646
1647
1648 mlx4_CLOSE_PORT(mdev->dev, priv->port);
1649
1650
1651 netif_tx_lock_bh(dev);
1652 if (detach)
1653 netif_device_detach(dev);
1654 netif_tx_stop_all_queues(dev);
1655 netif_tx_unlock_bh(dev);
1656
1657 netif_tx_disable(dev);
1658
1659
1660 priv->port_up = false;
1661
1662
1663 if (mdev->dev->caps.steering_mode ==
1664 MLX4_STEERING_MODE_DEVICE_MANAGED) {
1665 priv->flags &= ~(MLX4_EN_FLAG_PROMISC |
1666 MLX4_EN_FLAG_MC_PROMISC);
1667 mlx4_flow_steer_promisc_remove(mdev->dev,
1668 priv->port,
1669 MLX4_FS_ALL_DEFAULT);
1670 mlx4_flow_steer_promisc_remove(mdev->dev,
1671 priv->port,
1672 MLX4_FS_MC_DEFAULT);
1673 } else if (priv->flags & MLX4_EN_FLAG_PROMISC) {
1674 priv->flags &= ~MLX4_EN_FLAG_PROMISC;
1675
1676
1677 mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn,
1678 priv->port);
1679
1680
1681 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
1682 mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn,
1683 priv->port);
1684 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
1685 }
1686 }
1687
1688
1689 memset(&mc_list[10], 0xff, ETH_ALEN);
1690 mc_list[5] = priv->port;
1691 mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list,
1692 MLX4_PROT_ETH, priv->broadcast_id);
1693 list_for_each_entry(mclist, &priv->curr_list, list) {
1694 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
1695 mc_list[5] = priv->port;
1696 mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp,
1697 mc_list, MLX4_PROT_ETH, mclist->reg_id);
1698 }
1699 mlx4_en_clear_list(dev);
1700 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) {
1701 list_del(&mclist->list);
1702 kfree(mclist);
1703 }
1704
1705
1706 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG);
1707
1708
1709 if (mdev->dev->caps.steering_mode ==
1710 MLX4_STEERING_MODE_DEVICE_MANAGED) {
1711 ASSERT_RTNL();
1712 list_for_each_entry_safe(flow, tmp_flow,
1713 &priv->ethtool_list, list) {
1714 mlx4_flow_detach(mdev->dev, flow->id);
1715 list_del(&flow->list);
1716 }
1717 }
1718
1719 mlx4_en_destroy_drop_qp(priv);
1720
1721
1722 for (i = 0; i < priv->tx_ring_num; i++) {
1723 mlx4_en_deactivate_tx_ring(priv, &priv->tx_ring[i]);
1724 mlx4_en_deactivate_cq(priv, &priv->tx_cq[i]);
1725 }
1726 msleep(10);
1727
1728 for (i = 0; i < priv->tx_ring_num; i++)
1729 mlx4_en_free_tx_buf(dev, &priv->tx_ring[i]);
1730
1731
1732 mlx4_en_release_rss_steer(priv);
1733
1734
1735 mlx4_en_put_qp(priv);
1736 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAGS2_REASSIGN_MAC_EN))
1737 mdev->mac_removed[priv->port] = 1;
1738
1739
1740 for (i = 0; i < priv->rx_ring_num; i++) {
1741 struct mlx4_en_cq *cq = &priv->rx_cq[i];
1742
1743 local_bh_disable();
1744 while (!mlx4_en_cq_lock_napi(cq)) {
1745 pr_info("CQ %d locked\n", i);
1746 mdelay(1);
1747 }
1748 local_bh_enable();
1749
1750 while (test_bit(NAPI_STATE_SCHED, &cq->napi.state))
1751 msleep(1);
1752 mlx4_en_deactivate_rx_ring(priv, &priv->rx_ring[i]);
1753 mlx4_en_deactivate_cq(priv, cq);
1754 }
1755}
1756
1757static void mlx4_en_restart(struct work_struct *work)
1758{
1759 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1760 watchdog_task);
1761 struct mlx4_en_dev *mdev = priv->mdev;
1762 struct net_device *dev = priv->dev;
1763
1764 en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port);
1765
1766 mutex_lock(&mdev->state_lock);
1767 if (priv->port_up) {
1768 mlx4_en_stop_port(dev, 1);
1769 if (mlx4_en_start_port(dev))
1770 en_err(priv, "Failed restarting port %d\n", priv->port);
1771 }
1772 mutex_unlock(&mdev->state_lock);
1773}
1774
1775static void mlx4_en_clear_stats(struct net_device *dev)
1776{
1777 struct mlx4_en_priv *priv = netdev_priv(dev);
1778 struct mlx4_en_dev *mdev = priv->mdev;
1779 int i;
1780
1781 if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1))
1782 en_dbg(HW, priv, "Failed dumping statistics\n");
1783
1784 memset(&priv->stats, 0, sizeof(priv->stats));
1785 memset(&priv->pstats, 0, sizeof(priv->pstats));
1786 memset(&priv->pkstats, 0, sizeof(priv->pkstats));
1787 memset(&priv->port_stats, 0, sizeof(priv->port_stats));
1788
1789 for (i = 0; i < priv->tx_ring_num; i++) {
1790 priv->tx_ring[i].bytes = 0;
1791 priv->tx_ring[i].packets = 0;
1792 priv->tx_ring[i].tx_csum = 0;
1793 }
1794 for (i = 0; i < priv->rx_ring_num; i++) {
1795 priv->rx_ring[i].bytes = 0;
1796 priv->rx_ring[i].packets = 0;
1797 priv->rx_ring[i].csum_ok = 0;
1798 priv->rx_ring[i].csum_none = 0;
1799 }
1800}
1801
1802static int mlx4_en_open(struct net_device *dev)
1803{
1804 struct mlx4_en_priv *priv = netdev_priv(dev);
1805 struct mlx4_en_dev *mdev = priv->mdev;
1806 int err = 0;
1807
1808 mutex_lock(&mdev->state_lock);
1809
1810 if (!mdev->device_up) {
1811 en_err(priv, "Cannot open - device down/disabled\n");
1812 err = -EBUSY;
1813 goto out;
1814 }
1815
1816
1817 mlx4_en_clear_stats(dev);
1818
1819 err = mlx4_en_start_port(dev);
1820 if (err)
1821 en_err(priv, "Failed starting port:%d\n", priv->port);
1822
1823out:
1824 mutex_unlock(&mdev->state_lock);
1825 return err;
1826}
1827
1828
1829static int mlx4_en_close(struct net_device *dev)
1830{
1831 struct mlx4_en_priv *priv = netdev_priv(dev);
1832 struct mlx4_en_dev *mdev = priv->mdev;
1833
1834 en_dbg(IFDOWN, priv, "Close port called\n");
1835
1836 mutex_lock(&mdev->state_lock);
1837
1838 mlx4_en_stop_port(dev, 0);
1839 netif_carrier_off(dev);
1840
1841 mutex_unlock(&mdev->state_lock);
1842 return 0;
1843}
1844
1845void mlx4_en_free_resources(struct mlx4_en_priv *priv)
1846{
1847 int i;
1848
1849#ifdef CONFIG_RFS_ACCEL
1850 free_irq_cpu_rmap(priv->dev->rx_cpu_rmap);
1851 priv->dev->rx_cpu_rmap = NULL;
1852#endif
1853
1854 for (i = 0; i < priv->tx_ring_num; i++) {
1855 if (priv->tx_ring[i].tx_info)
1856 mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]);
1857 if (priv->tx_cq[i].buf)
1858 mlx4_en_destroy_cq(priv, &priv->tx_cq[i]);
1859 }
1860
1861 for (i = 0; i < priv->rx_ring_num; i++) {
1862 if (priv->rx_ring[i].rx_info)
1863 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i],
1864 priv->prof->rx_ring_size, priv->stride);
1865 if (priv->rx_cq[i].buf)
1866 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]);
1867 }
1868
1869 if (priv->base_tx_qpn) {
1870 mlx4_qp_release_range(priv->mdev->dev, priv->base_tx_qpn, priv->tx_ring_num);
1871 priv->base_tx_qpn = 0;
1872 }
1873}
1874
1875int mlx4_en_alloc_resources(struct mlx4_en_priv *priv)
1876{
1877 struct mlx4_en_port_profile *prof = priv->prof;
1878 int i;
1879 int err;
1880
1881 err = mlx4_qp_reserve_range(priv->mdev->dev, priv->tx_ring_num, 256, &priv->base_tx_qpn);
1882 if (err) {
1883 en_err(priv, "failed reserving range for TX rings\n");
1884 return err;
1885 }
1886
1887
1888 for (i = 0; i < priv->tx_ring_num; i++) {
1889 if (mlx4_en_create_cq(priv, &priv->tx_cq[i],
1890 prof->tx_ring_size, i, TX))
1891 goto err;
1892
1893 if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], priv->base_tx_qpn + i,
1894 prof->tx_ring_size, TXBB_SIZE))
1895 goto err;
1896 }
1897
1898
1899 for (i = 0; i < priv->rx_ring_num; i++) {
1900 if (mlx4_en_create_cq(priv, &priv->rx_cq[i],
1901 prof->rx_ring_size, i, RX))
1902 goto err;
1903
1904 if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i],
1905 prof->rx_ring_size, priv->stride))
1906 goto err;
1907 }
1908
1909#ifdef CONFIG_RFS_ACCEL
1910 if (priv->mdev->dev->caps.comp_pool) {
1911 priv->dev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->mdev->dev->caps.comp_pool);
1912 if (!priv->dev->rx_cpu_rmap)
1913 goto err;
1914 }
1915#endif
1916
1917 return 0;
1918
1919err:
1920 en_err(priv, "Failed to allocate NIC resources\n");
1921 return -ENOMEM;
1922}
1923
1924
1925void mlx4_en_destroy_netdev(struct net_device *dev)
1926{
1927 struct mlx4_en_priv *priv = netdev_priv(dev);
1928 struct mlx4_en_dev *mdev = priv->mdev;
1929
1930 en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port);
1931
1932
1933 if (priv->registered)
1934 unregister_netdev(dev);
1935
1936 if (priv->allocated)
1937 mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE);
1938
1939 cancel_delayed_work(&priv->stats_task);
1940 cancel_delayed_work(&priv->service_task);
1941
1942 flush_workqueue(mdev->workqueue);
1943
1944
1945 mutex_lock(&mdev->state_lock);
1946 mdev->pndev[priv->port] = NULL;
1947 mutex_unlock(&mdev->state_lock);
1948
1949 mlx4_en_free_resources(priv);
1950
1951 kfree(priv->tx_ring);
1952 kfree(priv->tx_cq);
1953
1954 free_netdev(dev);
1955}
1956
1957static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu)
1958{
1959 struct mlx4_en_priv *priv = netdev_priv(dev);
1960 struct mlx4_en_dev *mdev = priv->mdev;
1961 int err = 0;
1962
1963 en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n",
1964 dev->mtu, new_mtu);
1965
1966 if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) {
1967 en_err(priv, "Bad MTU size:%d.\n", new_mtu);
1968 return -EPERM;
1969 }
1970 dev->mtu = new_mtu;
1971
1972 if (netif_running(dev)) {
1973 mutex_lock(&mdev->state_lock);
1974 if (!mdev->device_up) {
1975
1976
1977 en_dbg(DRV, priv, "Change MTU called with card down!?\n");
1978 } else {
1979 mlx4_en_stop_port(dev, 1);
1980 err = mlx4_en_start_port(dev);
1981 if (err) {
1982 en_err(priv, "Failed restarting port:%d\n",
1983 priv->port);
1984 queue_work(mdev->workqueue, &priv->watchdog_task);
1985 }
1986 }
1987 mutex_unlock(&mdev->state_lock);
1988 }
1989 return 0;
1990}
1991
1992static int mlx4_en_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
1993{
1994 struct mlx4_en_priv *priv = netdev_priv(dev);
1995 struct mlx4_en_dev *mdev = priv->mdev;
1996 struct hwtstamp_config config;
1997
1998 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1999 return -EFAULT;
2000
2001
2002 if (config.flags)
2003 return -EINVAL;
2004
2005
2006 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS))
2007 return -EINVAL;
2008
2009
2010 switch (config.tx_type) {
2011 case HWTSTAMP_TX_OFF:
2012 case HWTSTAMP_TX_ON:
2013 break;
2014 default:
2015 return -ERANGE;
2016 }
2017
2018
2019 switch (config.rx_filter) {
2020 case HWTSTAMP_FILTER_NONE:
2021 break;
2022 case HWTSTAMP_FILTER_ALL:
2023 case HWTSTAMP_FILTER_SOME:
2024 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2025 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2026 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2027 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2028 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2029 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2030 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2031 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2032 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2033 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2034 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2035 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2036 config.rx_filter = HWTSTAMP_FILTER_ALL;
2037 break;
2038 default:
2039 return -ERANGE;
2040 }
2041
2042 if (mlx4_en_timestamp_config(dev, config.tx_type, config.rx_filter)) {
2043 config.tx_type = HWTSTAMP_TX_OFF;
2044 config.rx_filter = HWTSTAMP_FILTER_NONE;
2045 }
2046
2047 return copy_to_user(ifr->ifr_data, &config,
2048 sizeof(config)) ? -EFAULT : 0;
2049}
2050
2051static int mlx4_en_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2052{
2053 switch (cmd) {
2054 case SIOCSHWTSTAMP:
2055 return mlx4_en_hwtstamp_ioctl(dev, ifr);
2056 default:
2057 return -EOPNOTSUPP;
2058 }
2059}
2060
2061static int mlx4_en_set_features(struct net_device *netdev,
2062 netdev_features_t features)
2063{
2064 struct mlx4_en_priv *priv = netdev_priv(netdev);
2065
2066 if (features & NETIF_F_LOOPBACK)
2067 priv->ctrl_flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
2068 else
2069 priv->ctrl_flags &=
2070 cpu_to_be32(~MLX4_WQE_CTRL_FORCE_LOOPBACK);
2071
2072 mlx4_en_update_loopback_state(netdev, features);
2073
2074 return 0;
2075
2076}
2077
2078static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac)
2079{
2080 struct mlx4_en_priv *en_priv = netdev_priv(dev);
2081 struct mlx4_en_dev *mdev = en_priv->mdev;
2082 u64 mac_u64 = mlx4_en_mac_to_u64(mac);
2083
2084 if (!is_valid_ether_addr(mac))
2085 return -EINVAL;
2086
2087 return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac_u64);
2088}
2089
2090static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2091{
2092 struct mlx4_en_priv *en_priv = netdev_priv(dev);
2093 struct mlx4_en_dev *mdev = en_priv->mdev;
2094
2095 return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos);
2096}
2097
2098static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2099{
2100 struct mlx4_en_priv *en_priv = netdev_priv(dev);
2101 struct mlx4_en_dev *mdev = en_priv->mdev;
2102
2103 return mlx4_set_vf_spoofchk(mdev->dev, en_priv->port, vf, setting);
2104}
2105
2106static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf)
2107{
2108 struct mlx4_en_priv *en_priv = netdev_priv(dev);
2109 struct mlx4_en_dev *mdev = en_priv->mdev;
2110
2111 return mlx4_get_vf_config(mdev->dev, en_priv->port, vf, ivf);
2112}
2113
2114static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state)
2115{
2116 struct mlx4_en_priv *en_priv = netdev_priv(dev);
2117 struct mlx4_en_dev *mdev = en_priv->mdev;
2118
2119 return mlx4_set_vf_link_state(mdev->dev, en_priv->port, vf, link_state);
2120}
2121static const struct net_device_ops mlx4_netdev_ops = {
2122 .ndo_open = mlx4_en_open,
2123 .ndo_stop = mlx4_en_close,
2124 .ndo_start_xmit = mlx4_en_xmit,
2125 .ndo_select_queue = mlx4_en_select_queue,
2126 .ndo_get_stats = mlx4_en_get_stats,
2127 .ndo_set_rx_mode = mlx4_en_set_rx_mode,
2128 .ndo_set_mac_address = mlx4_en_set_mac,
2129 .ndo_validate_addr = eth_validate_addr,
2130 .ndo_change_mtu = mlx4_en_change_mtu,
2131 .ndo_do_ioctl = mlx4_en_ioctl,
2132 .ndo_tx_timeout = mlx4_en_tx_timeout,
2133 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid,
2134 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid,
2135#ifdef CONFIG_NET_POLL_CONTROLLER
2136 .ndo_poll_controller = mlx4_en_netpoll,
2137#endif
2138 .ndo_set_features = mlx4_en_set_features,
2139 .ndo_setup_tc = mlx4_en_setup_tc,
2140#ifdef CONFIG_RFS_ACCEL
2141 .ndo_rx_flow_steer = mlx4_en_filter_rfs,
2142#endif
2143#ifdef CONFIG_NET_RX_BUSY_POLL
2144 .ndo_busy_poll = mlx4_en_low_latency_recv,
2145#endif
2146};
2147
2148static const struct net_device_ops mlx4_netdev_ops_master = {
2149 .ndo_open = mlx4_en_open,
2150 .ndo_stop = mlx4_en_close,
2151 .ndo_start_xmit = mlx4_en_xmit,
2152 .ndo_select_queue = mlx4_en_select_queue,
2153 .ndo_get_stats = mlx4_en_get_stats,
2154 .ndo_set_rx_mode = mlx4_en_set_rx_mode,
2155 .ndo_set_mac_address = mlx4_en_set_mac,
2156 .ndo_validate_addr = eth_validate_addr,
2157 .ndo_change_mtu = mlx4_en_change_mtu,
2158 .ndo_tx_timeout = mlx4_en_tx_timeout,
2159 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid,
2160 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid,
2161 .ndo_set_vf_mac = mlx4_en_set_vf_mac,
2162 .ndo_set_vf_vlan = mlx4_en_set_vf_vlan,
2163 .ndo_set_vf_spoofchk = mlx4_en_set_vf_spoofchk,
2164 .ndo_set_vf_link_state = mlx4_en_set_vf_link_state,
2165 .ndo_get_vf_config = mlx4_en_get_vf_config,
2166#ifdef CONFIG_NET_POLL_CONTROLLER
2167 .ndo_poll_controller = mlx4_en_netpoll,
2168#endif
2169 .ndo_set_features = mlx4_en_set_features,
2170 .ndo_setup_tc = mlx4_en_setup_tc,
2171#ifdef CONFIG_RFS_ACCEL
2172 .ndo_rx_flow_steer = mlx4_en_filter_rfs,
2173#endif
2174};
2175
2176int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
2177 struct mlx4_en_port_profile *prof)
2178{
2179 struct net_device *dev;
2180 struct mlx4_en_priv *priv;
2181 int i;
2182 int err;
2183 u64 mac_u64;
2184
2185 dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv),
2186 MAX_TX_RINGS, MAX_RX_RINGS);
2187 if (dev == NULL)
2188 return -ENOMEM;
2189
2190 netif_set_real_num_tx_queues(dev, prof->tx_ring_num);
2191 netif_set_real_num_rx_queues(dev, prof->rx_ring_num);
2192
2193 SET_NETDEV_DEV(dev, &mdev->dev->pdev->dev);
2194 dev->dev_id = port - 1;
2195
2196
2197
2198
2199
2200 priv = netdev_priv(dev);
2201 memset(priv, 0, sizeof(struct mlx4_en_priv));
2202 priv->dev = dev;
2203 priv->mdev = mdev;
2204 priv->ddev = &mdev->pdev->dev;
2205 priv->prof = prof;
2206 priv->port = port;
2207 priv->port_up = false;
2208 priv->flags = prof->flags;
2209 priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE |
2210 MLX4_WQE_CTRL_SOLICITED);
2211 priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up;
2212 priv->tx_ring_num = prof->tx_ring_num;
2213
2214 priv->tx_ring = kzalloc(sizeof(struct mlx4_en_tx_ring) * MAX_TX_RINGS,
2215 GFP_KERNEL);
2216 if (!priv->tx_ring) {
2217 err = -ENOMEM;
2218 goto out;
2219 }
2220 priv->tx_cq = kzalloc(sizeof(struct mlx4_en_cq) * MAX_TX_RINGS,
2221 GFP_KERNEL);
2222 if (!priv->tx_cq) {
2223 err = -ENOMEM;
2224 goto out;
2225 }
2226 priv->rx_ring_num = prof->rx_ring_num;
2227 priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0;
2228 priv->mac_index = -1;
2229 priv->msg_enable = MLX4_EN_MSG_LEVEL;
2230 spin_lock_init(&priv->stats_lock);
2231 INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode);
2232 INIT_WORK(&priv->watchdog_task, mlx4_en_restart);
2233 INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate);
2234 INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats);
2235 INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task);
2236#ifdef CONFIG_MLX4_EN_DCB
2237 if (!mlx4_is_slave(priv->mdev->dev)) {
2238 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) {
2239 dev->dcbnl_ops = &mlx4_en_dcbnl_ops;
2240 } else {
2241 en_info(priv, "enabling only PFC DCB ops\n");
2242 dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops;
2243 }
2244 }
2245#endif
2246
2247 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i)
2248 INIT_HLIST_HEAD(&priv->mac_hash[i]);
2249
2250
2251 priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port];
2252
2253
2254 dev->addr_len = ETH_ALEN;
2255 mlx4_en_u64_to_mac(dev->dev_addr, mdev->dev->caps.def_mac[priv->port]);
2256 if (!is_valid_ether_addr(dev->dev_addr)) {
2257 if (mlx4_is_slave(priv->mdev->dev)) {
2258 eth_hw_addr_random(dev);
2259 en_warn(priv, "Assigned random MAC address %pM\n", dev->dev_addr);
2260 mac_u64 = mlx4_en_mac_to_u64(dev->dev_addr);
2261 mdev->dev->caps.def_mac[priv->port] = mac_u64;
2262 } else {
2263 en_err(priv, "Port: %d, invalid mac burned: %pM, quiting\n",
2264 priv->port, dev->dev_addr);
2265 err = -EINVAL;
2266 goto out;
2267 }
2268 }
2269
2270 memcpy(priv->prev_mac, dev->dev_addr, sizeof(priv->prev_mac));
2271
2272 priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
2273 DS_SIZE * MLX4_EN_MAX_RX_FRAGS);
2274 err = mlx4_en_alloc_resources(priv);
2275 if (err)
2276 goto out;
2277
2278#ifdef CONFIG_RFS_ACCEL
2279 INIT_LIST_HEAD(&priv->filters);
2280 spin_lock_init(&priv->filters_lock);
2281#endif
2282
2283
2284 priv->hwtstamp_config.flags = 0;
2285 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF;
2286 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
2287
2288
2289 err = mlx4_alloc_hwq_res(mdev->dev, &priv->res,
2290 MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE);
2291 if (err) {
2292 en_err(priv, "Failed to allocate page for rx qps\n");
2293 goto out;
2294 }
2295 priv->allocated = 1;
2296
2297
2298
2299
2300 if (mlx4_is_master(priv->mdev->dev))
2301 dev->netdev_ops = &mlx4_netdev_ops_master;
2302 else
2303 dev->netdev_ops = &mlx4_netdev_ops;
2304 dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT;
2305 netif_set_real_num_tx_queues(dev, priv->tx_ring_num);
2306 netif_set_real_num_rx_queues(dev, priv->rx_ring_num);
2307
2308 SET_ETHTOOL_OPS(dev, &mlx4_en_ethtool_ops);
2309
2310
2311
2312
2313 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2314 if (mdev->LSO_support)
2315 dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2316
2317 dev->vlan_features = dev->hw_features;
2318
2319 dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH;
2320 dev->features = dev->hw_features | NETIF_F_HIGHDMA |
2321 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2322 NETIF_F_HW_VLAN_CTAG_FILTER;
2323 dev->hw_features |= NETIF_F_LOOPBACK;
2324
2325 if (mdev->dev->caps.steering_mode ==
2326 MLX4_STEERING_MODE_DEVICE_MANAGED)
2327 dev->hw_features |= NETIF_F_NTUPLE;
2328
2329 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0)
2330 dev->priv_flags |= IFF_UNICAST_FLT;
2331
2332 mdev->pndev[port] = dev;
2333
2334 netif_carrier_off(dev);
2335 mlx4_en_set_default_moderation(priv);
2336
2337 err = register_netdev(dev);
2338 if (err) {
2339 en_err(priv, "Netdev registration failed for port %d\n", port);
2340 goto out;
2341 }
2342 priv->registered = 1;
2343
2344 en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num);
2345 en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num);
2346
2347 mlx4_en_update_loopback_state(priv->dev, priv->dev->features);
2348
2349
2350 mlx4_en_calc_rx_buf(dev);
2351 err = mlx4_SET_PORT_general(mdev->dev, priv->port,
2352 priv->rx_skb_size + ETH_FCS_LEN,
2353 prof->tx_pause, prof->tx_ppp,
2354 prof->rx_pause, prof->rx_ppp);
2355 if (err) {
2356 en_err(priv, "Failed setting port general configurations "
2357 "for port %d, with error %d\n", priv->port, err);
2358 goto out;
2359 }
2360
2361
2362 en_warn(priv, "Initializing port\n");
2363 err = mlx4_INIT_PORT(mdev->dev, priv->port);
2364 if (err) {
2365 en_err(priv, "Failed Initializing port\n");
2366 goto out;
2367 }
2368 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY);
2369
2370 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)
2371 queue_delayed_work(mdev->workqueue, &priv->service_task,
2372 SERVICE_TASK_DELAY);
2373
2374 return 0;
2375
2376out:
2377 mlx4_en_destroy_netdev(dev);
2378 return err;
2379}
2380
2381
