1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include "fm10k.h"
22#include <linux/vmalloc.h>
23#include <net/udp_tunnel.h>
24
25
26
27
28
29
30
31int fm10k_setup_tx_resources(struct fm10k_ring *tx_ring)
32{
33 struct device *dev = tx_ring->dev;
34 int size;
35
36 size = sizeof(struct fm10k_tx_buffer) * tx_ring->count;
37
38 tx_ring->tx_buffer = vzalloc(size);
39 if (!tx_ring->tx_buffer)
40 goto err;
41
42 u64_stats_init(&tx_ring->syncp);
43
44
45 tx_ring->size = tx_ring->count * sizeof(struct fm10k_tx_desc);
46 tx_ring->size = ALIGN(tx_ring->size, 4096);
47
48 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
49 &tx_ring->dma, GFP_KERNEL);
50 if (!tx_ring->desc)
51 goto err;
52
53 return 0;
54
55err:
56 vfree(tx_ring->tx_buffer);
57 tx_ring->tx_buffer = NULL;
58 return -ENOMEM;
59}
60
61
62
63
64
65
66
67
68
69
70
71static int fm10k_setup_all_tx_resources(struct fm10k_intfc *interface)
72{
73 int i, err = 0;
74
75 for (i = 0; i < interface->num_tx_queues; i++) {
76 err = fm10k_setup_tx_resources(interface->tx_ring[i]);
77 if (!err)
78 continue;
79
80 netif_err(interface, probe, interface->netdev,
81 "Allocation for Tx Queue %u failed\n", i);
82 goto err_setup_tx;
83 }
84
85 return 0;
86err_setup_tx:
87
88 while (i--)
89 fm10k_free_tx_resources(interface->tx_ring[i]);
90 return err;
91}
92
93
94
95
96
97
98
99int fm10k_setup_rx_resources(struct fm10k_ring *rx_ring)
100{
101 struct device *dev = rx_ring->dev;
102 int size;
103
104 size = sizeof(struct fm10k_rx_buffer) * rx_ring->count;
105
106 rx_ring->rx_buffer = vzalloc(size);
107 if (!rx_ring->rx_buffer)
108 goto err;
109
110 u64_stats_init(&rx_ring->syncp);
111
112
113 rx_ring->size = rx_ring->count * sizeof(union fm10k_rx_desc);
114 rx_ring->size = ALIGN(rx_ring->size, 4096);
115
116 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
117 &rx_ring->dma, GFP_KERNEL);
118 if (!rx_ring->desc)
119 goto err;
120
121 return 0;
122err:
123 vfree(rx_ring->rx_buffer);
124 rx_ring->rx_buffer = NULL;
125 return -ENOMEM;
126}
127
128
129
130
131
132
133
134
135
136
137
138static int fm10k_setup_all_rx_resources(struct fm10k_intfc *interface)
139{
140 int i, err = 0;
141
142 for (i = 0; i < interface->num_rx_queues; i++) {
143 err = fm10k_setup_rx_resources(interface->rx_ring[i]);
144 if (!err)
145 continue;
146
147 netif_err(interface, probe, interface->netdev,
148 "Allocation for Rx Queue %u failed\n", i);
149 goto err_setup_rx;
150 }
151
152 return 0;
153err_setup_rx:
154
155 while (i--)
156 fm10k_free_rx_resources(interface->rx_ring[i]);
157 return err;
158}
159
160void fm10k_unmap_and_free_tx_resource(struct fm10k_ring *ring,
161 struct fm10k_tx_buffer *tx_buffer)
162{
163 if (tx_buffer->skb) {
164 dev_kfree_skb_any(tx_buffer->skb);
165 if (dma_unmap_len(tx_buffer, len))
166 dma_unmap_single(ring->dev,
167 dma_unmap_addr(tx_buffer, dma),
168 dma_unmap_len(tx_buffer, len),
169 DMA_TO_DEVICE);
170 } else if (dma_unmap_len(tx_buffer, len)) {
171 dma_unmap_page(ring->dev,
172 dma_unmap_addr(tx_buffer, dma),
173 dma_unmap_len(tx_buffer, len),
174 DMA_TO_DEVICE);
175 }
176 tx_buffer->next_to_watch = NULL;
177 tx_buffer->skb = NULL;
178 dma_unmap_len_set(tx_buffer, len, 0);
179
180}
181
182
183
184
185
186static void fm10k_clean_tx_ring(struct fm10k_ring *tx_ring)
187{
188 struct fm10k_tx_buffer *tx_buffer;
189 unsigned long size;
190 u16 i;
191
192
193 if (!tx_ring->tx_buffer)
194 return;
195
196
197 for (i = 0; i < tx_ring->count; i++) {
198 tx_buffer = &tx_ring->tx_buffer[i];
199 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer);
200 }
201
202
203 netdev_tx_reset_queue(txring_txq(tx_ring));
204
205 size = sizeof(struct fm10k_tx_buffer) * tx_ring->count;
206 memset(tx_ring->tx_buffer, 0, size);
207
208
209 memset(tx_ring->desc, 0, tx_ring->size);
210}
211
212
213
214
215
216
217
218void fm10k_free_tx_resources(struct fm10k_ring *tx_ring)
219{
220 fm10k_clean_tx_ring(tx_ring);
221
222 vfree(tx_ring->tx_buffer);
223 tx_ring->tx_buffer = NULL;
224
225
226 if (!tx_ring->desc)
227 return;
228
229 dma_free_coherent(tx_ring->dev, tx_ring->size,
230 tx_ring->desc, tx_ring->dma);
231 tx_ring->desc = NULL;
232}
233
234
235
236
237
238void fm10k_clean_all_tx_rings(struct fm10k_intfc *interface)
239{
240 int i;
241
242 for (i = 0; i < interface->num_tx_queues; i++)
243 fm10k_clean_tx_ring(interface->tx_ring[i]);
244}
245
246
247
248
249
250
251
252static void fm10k_free_all_tx_resources(struct fm10k_intfc *interface)
253{
254 int i = interface->num_tx_queues;
255
256 while (i--)
257 fm10k_free_tx_resources(interface->tx_ring[i]);
258}
259
260
261
262
263
264static void fm10k_clean_rx_ring(struct fm10k_ring *rx_ring)
265{
266 unsigned long size;
267 u16 i;
268
269 if (!rx_ring->rx_buffer)
270 return;
271
272 if (rx_ring->skb)
273 dev_kfree_skb(rx_ring->skb);
274 rx_ring->skb = NULL;
275
276
277 for (i = 0; i < rx_ring->count; i++) {
278 struct fm10k_rx_buffer *buffer = &rx_ring->rx_buffer[i];
279
280 if (!buffer->page)
281 continue;
282
283 dma_unmap_page(rx_ring->dev, buffer->dma,
284 PAGE_SIZE, DMA_FROM_DEVICE);
285 __free_page(buffer->page);
286
287 buffer->page = NULL;
288 }
289
290 size = sizeof(struct fm10k_rx_buffer) * rx_ring->count;
291 memset(rx_ring->rx_buffer, 0, size);
292
293
294 memset(rx_ring->desc, 0, rx_ring->size);
295
296 rx_ring->next_to_alloc = 0;
297 rx_ring->next_to_clean = 0;
298 rx_ring->next_to_use = 0;
299}
300
301
302
303
304
305
306
307void fm10k_free_rx_resources(struct fm10k_ring *rx_ring)
308{
309 fm10k_clean_rx_ring(rx_ring);
310
311 vfree(rx_ring->rx_buffer);
312 rx_ring->rx_buffer = NULL;
313
314
315 if (!rx_ring->desc)
316 return;
317
318 dma_free_coherent(rx_ring->dev, rx_ring->size,
319 rx_ring->desc, rx_ring->dma);
320
321 rx_ring->desc = NULL;
322}
323
324
325
326
327
328void fm10k_clean_all_rx_rings(struct fm10k_intfc *interface)
329{
330 int i;
331
332 for (i = 0; i < interface->num_rx_queues; i++)
333 fm10k_clean_rx_ring(interface->rx_ring[i]);
334}
335
336
337
338
339
340
341
342static void fm10k_free_all_rx_resources(struct fm10k_intfc *interface)
343{
344 int i = interface->num_rx_queues;
345
346 while (i--)
347 fm10k_free_rx_resources(interface->rx_ring[i]);
348}
349
350
351
352
353
354
355
356static void fm10k_request_glort_range(struct fm10k_intfc *interface)
357{
358 struct fm10k_hw *hw = &interface->hw;
359 u16 mask = (~hw->mac.dglort_map) >> FM10K_DGLORTMAP_MASK_SHIFT;
360
361
362 interface->glort = hw->mac.dglort_map & FM10K_DGLORTMAP_NONE;
363 interface->glort_count = 0;
364
365
366 if (hw->mac.dglort_map == FM10K_DGLORTMAP_NONE)
367 return;
368
369
370
371
372
373
374 if (mask <= hw->iov.total_vfs) {
375 interface->glort_count = 1;
376 interface->glort += mask;
377 } else if (mask < 64) {
378 interface->glort_count = (mask + 1) / 2;
379 interface->glort += interface->glort_count;
380 } else {
381 interface->glort_count = mask - 63;
382 interface->glort += 64;
383 }
384}
385
386
387
388
389
390
391
392static void fm10k_free_udp_port_info(struct fm10k_intfc *interface)
393{
394 struct fm10k_udp_port *port;
395
396
397 port = list_first_entry_or_null(&interface->vxlan_port,
398 struct fm10k_udp_port, list);
399 while (port) {
400 list_del(&port->list);
401 kfree(port);
402 port = list_first_entry_or_null(&interface->vxlan_port,
403 struct fm10k_udp_port,
404 list);
405 }
406
407
408 port = list_first_entry_or_null(&interface->geneve_port,
409 struct fm10k_udp_port, list);
410 while (port) {
411 list_del(&port->list);
412 kfree(port);
413 port = list_first_entry_or_null(&interface->vxlan_port,
414 struct fm10k_udp_port,
415 list);
416 }
417}
418
419
420
421
422
423
424
425static void fm10k_restore_udp_port_info(struct fm10k_intfc *interface)
426{
427 struct fm10k_hw *hw = &interface->hw;
428 struct fm10k_udp_port *port;
429
430
431 if (hw->mac.type != fm10k_mac_pf)
432 return;
433
434 port = list_first_entry_or_null(&interface->vxlan_port,
435 struct fm10k_udp_port, list);
436
437
438 fm10k_write_reg(hw, FM10K_TUNNEL_CFG,
439 (port ? ntohs(port->port) : 0) |
440 (ETH_P_TEB << FM10K_TUNNEL_CFG_NVGRE_SHIFT));
441
442 port = list_first_entry_or_null(&interface->geneve_port,
443 struct fm10k_udp_port, list);
444
445
446 fm10k_write_reg(hw, FM10K_TUNNEL_CFG_GENEVE,
447 (port ? ntohs(port->port) : 0));
448}
449
450static struct fm10k_udp_port *
451fm10k_remove_tunnel_port(struct list_head *ports,
452 struct udp_tunnel_info *ti)
453{
454 struct fm10k_udp_port *port;
455
456 list_for_each_entry(port, ports, list) {
457 if ((port->port == ti->port) &&
458 (port->sa_family == ti->sa_family)) {
459 list_del(&port->list);
460 return port;
461 }
462 }
463
464 return NULL;
465}
466
467static void fm10k_insert_tunnel_port(struct list_head *ports,
468 struct udp_tunnel_info *ti)
469{
470 struct fm10k_udp_port *port;
471
472
473
474
475 port = fm10k_remove_tunnel_port(ports, ti);
476 if (!port) {
477 port = kmalloc(sizeof(*port), GFP_ATOMIC);
478 if (!port)
479 return;
480 port->port = ti->port;
481 port->sa_family = ti->sa_family;
482 }
483
484 list_add_tail(&port->list, ports);
485}
486
487
488
489
490
491
492
493
494
495
496static void fm10k_udp_tunnel_add(struct net_device *dev,
497 struct udp_tunnel_info *ti)
498{
499 struct fm10k_intfc *interface = netdev_priv(dev);
500
501
502 if (interface->hw.mac.type != fm10k_mac_pf)
503 return;
504
505 switch (ti->type) {
506 case UDP_TUNNEL_TYPE_VXLAN:
507 fm10k_insert_tunnel_port(&interface->vxlan_port, ti);
508 break;
509 case UDP_TUNNEL_TYPE_GENEVE:
510 fm10k_insert_tunnel_port(&interface->geneve_port, ti);
511 break;
512 default:
513 return;
514 }
515
516 fm10k_restore_udp_port_info(interface);
517}
518
519
520
521
522
523
524
525
526
527
528static void fm10k_udp_tunnel_del(struct net_device *dev,
529 struct udp_tunnel_info *ti)
530{
531 struct fm10k_intfc *interface = netdev_priv(dev);
532 struct fm10k_udp_port *port = NULL;
533
534 if (interface->hw.mac.type != fm10k_mac_pf)
535 return;
536
537 switch (ti->type) {
538 case UDP_TUNNEL_TYPE_VXLAN:
539 port = fm10k_remove_tunnel_port(&interface->vxlan_port, ti);
540 break;
541 case UDP_TUNNEL_TYPE_GENEVE:
542 port = fm10k_remove_tunnel_port(&interface->geneve_port, ti);
543 break;
544 default:
545 return;
546 }
547
548
549 kfree(port);
550
551 fm10k_restore_udp_port_info(interface);
552}
553
554
555
556
557
558
559
560
561
562
563
564
565
566int fm10k_open(struct net_device *netdev)
567{
568 struct fm10k_intfc *interface = netdev_priv(netdev);
569 int err;
570
571
572 err = fm10k_setup_all_tx_resources(interface);
573 if (err)
574 goto err_setup_tx;
575
576
577 err = fm10k_setup_all_rx_resources(interface);
578 if (err)
579 goto err_setup_rx;
580
581
582 err = fm10k_qv_request_irq(interface);
583 if (err)
584 goto err_req_irq;
585
586
587 fm10k_request_glort_range(interface);
588
589
590 err = netif_set_real_num_tx_queues(netdev,
591 interface->num_tx_queues);
592 if (err)
593 goto err_set_queues;
594
595 err = netif_set_real_num_rx_queues(netdev,
596 interface->num_rx_queues);
597 if (err)
598 goto err_set_queues;
599
600 udp_tunnel_get_rx_info(netdev);
601
602 fm10k_up(interface);
603
604 return 0;
605
606err_set_queues:
607 fm10k_qv_free_irq(interface);
608err_req_irq:
609 fm10k_free_all_rx_resources(interface);
610err_setup_rx:
611 fm10k_free_all_tx_resources(interface);
612err_setup_tx:
613 return err;
614}
615
616
617
618
619
620
621
622
623
624
625
626
627int fm10k_close(struct net_device *netdev)
628{
629 struct fm10k_intfc *interface = netdev_priv(netdev);
630
631 fm10k_down(interface);
632
633 fm10k_qv_free_irq(interface);
634
635 fm10k_free_udp_port_info(interface);
636
637 fm10k_free_all_tx_resources(interface);
638 fm10k_free_all_rx_resources(interface);
639
640 return 0;
641}
642
643static netdev_tx_t fm10k_xmit_frame(struct sk_buff *skb, struct net_device *dev)
644{
645 struct fm10k_intfc *interface = netdev_priv(dev);
646 int num_tx_queues = READ_ONCE(interface->num_tx_queues);
647 unsigned int r_idx = skb->queue_mapping;
648 int err;
649
650 if (!num_tx_queues)
651 return NETDEV_TX_BUSY;
652
653 if ((skb->protocol == htons(ETH_P_8021Q)) &&
654 !skb_vlan_tag_present(skb)) {
655
656
657
658 struct vlan_hdr *vhdr;
659 __be16 proto;
660
661
662 skb = skb_share_check(skb, GFP_ATOMIC);
663 if (!skb)
664 return NETDEV_TX_OK;
665
666
667 if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN)))
668 return NETDEV_TX_OK;
669
670
671 err = skb_cow_head(skb, 0);
672 if (err) {
673 dev_kfree_skb(skb);
674 return NETDEV_TX_OK;
675 }
676
677
678 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
679
680
681 __vlan_hwaccel_put_tag(skb,
682 htons(ETH_P_8021Q),
683 ntohs(vhdr->h_vlan_TCI));
684 proto = vhdr->h_vlan_encapsulated_proto;
685 skb->protocol = (ntohs(proto) >= 1536) ? proto :
686 htons(ETH_P_802_2);
687
688
689 memmove(skb->data + VLAN_HLEN, skb->data, 12);
690 __skb_pull(skb, VLAN_HLEN);
691 skb_reset_mac_header(skb);
692 }
693
694
695
696
697 if (unlikely(skb->len < 17)) {
698 int pad_len = 17 - skb->len;
699
700 if (skb_pad(skb, pad_len))
701 return NETDEV_TX_OK;
702 __skb_put(skb, pad_len);
703 }
704
705 if (r_idx >= num_tx_queues)
706 r_idx %= num_tx_queues;
707
708 err = fm10k_xmit_frame_ring(skb, interface->tx_ring[r_idx]);
709
710 return err;
711}
712
713
714
715
716
717static void fm10k_tx_timeout(struct net_device *netdev)
718{
719 struct fm10k_intfc *interface = netdev_priv(netdev);
720 bool real_tx_hang = false;
721 int i;
722
723#define TX_TIMEO_LIMIT 16000
724 for (i = 0; i < interface->num_tx_queues; i++) {
725 struct fm10k_ring *tx_ring = interface->tx_ring[i];
726
727 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring))
728 real_tx_hang = true;
729 }
730
731 if (real_tx_hang) {
732 fm10k_tx_timeout_reset(interface);
733 } else {
734 netif_info(interface, drv, netdev,
735 "Fake Tx hang detected with timeout of %d seconds\n",
736 netdev->watchdog_timeo / HZ);
737
738
739 if (netdev->watchdog_timeo < TX_TIMEO_LIMIT)
740 netdev->watchdog_timeo *= 2;
741 }
742}
743
744
745
746
747
748
749
750
751
752
753
754static bool fm10k_host_mbx_ready(struct fm10k_intfc *interface)
755{
756 struct fm10k_hw *hw = &interface->hw;
757
758 return (hw->mac.type == fm10k_mac_vf || interface->host_ready);
759}
760
761
762
763
764
765
766
767
768
769
770
771
772
773int fm10k_queue_vlan_request(struct fm10k_intfc *interface,
774 u32 vid, u8 vsi, bool set)
775{
776 struct fm10k_macvlan_request *request;
777 unsigned long flags;
778
779
780
781
782 request = kzalloc(sizeof(*request), GFP_ATOMIC);
783 if (!request)
784 return -ENOMEM;
785
786 request->type = FM10K_VLAN_REQUEST;
787 request->vlan.vid = vid;
788 request->vlan.vsi = vsi;
789 request->set = set;
790
791 spin_lock_irqsave(&interface->macvlan_lock, flags);
792 list_add_tail(&request->list, &interface->macvlan_requests);
793 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
794
795 fm10k_macvlan_schedule(interface);
796
797 return 0;
798}
799
800
801
802
803
804
805
806
807
808
809
810
811
812int fm10k_queue_mac_request(struct fm10k_intfc *interface, u16 glort,
813 const unsigned char *addr, u16 vid, bool set)
814{
815 struct fm10k_macvlan_request *request;
816 unsigned long flags;
817
818
819
820
821 request = kzalloc(sizeof(*request), GFP_ATOMIC);
822 if (!request)
823 return -ENOMEM;
824
825 if (is_multicast_ether_addr(addr))
826 request->type = FM10K_MC_MAC_REQUEST;
827 else
828 request->type = FM10K_UC_MAC_REQUEST;
829
830 ether_addr_copy(request->mac.addr, addr);
831 request->mac.glort = glort;
832 request->mac.vid = vid;
833 request->set = set;
834
835 spin_lock_irqsave(&interface->macvlan_lock, flags);
836 list_add_tail(&request->list, &interface->macvlan_requests);
837 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
838
839 fm10k_macvlan_schedule(interface);
840
841 return 0;
842}
843
844
845
846
847
848
849
850
851
852
853void fm10k_clear_macvlan_queue(struct fm10k_intfc *interface,
854 u16 glort, bool vlans)
855
856{
857 struct fm10k_macvlan_request *r, *tmp;
858 unsigned long flags;
859
860 spin_lock_irqsave(&interface->macvlan_lock, flags);
861
862
863 list_for_each_entry_safe(r, tmp, &interface->macvlan_requests, list) {
864 switch (r->type) {
865 case FM10K_MC_MAC_REQUEST:
866 case FM10K_UC_MAC_REQUEST:
867
868 if (r->mac.glort != glort)
869 break;
870
871 case FM10K_VLAN_REQUEST:
872 if (vlans) {
873 list_del(&r->list);
874 kfree(r);
875 }
876 break;
877 }
878 }
879
880 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
881}
882
883static int fm10k_uc_vlan_unsync(struct net_device *netdev,
884 const unsigned char *uc_addr)
885{
886 struct fm10k_intfc *interface = netdev_priv(netdev);
887 u16 glort = interface->glort;
888 u16 vid = interface->vid;
889 bool set = !!(vid / VLAN_N_VID);
890 int err = -EHOSTDOWN;
891
892
893 vid &= VLAN_N_VID - 1;
894
895 err = fm10k_queue_mac_request(interface, glort, uc_addr, vid, set);
896 if (err)
897 return err;
898
899
900 return 1;
901}
902
903static int fm10k_mc_vlan_unsync(struct net_device *netdev,
904 const unsigned char *mc_addr)
905{
906 struct fm10k_intfc *interface = netdev_priv(netdev);
907 u16 glort = interface->glort;
908 u16 vid = interface->vid;
909 bool set = !!(vid / VLAN_N_VID);
910 int err = -EHOSTDOWN;
911
912
913 vid &= VLAN_N_VID - 1;
914
915 err = fm10k_queue_mac_request(interface, glort, mc_addr, vid, set);
916 if (err)
917 return err;
918
919
920 return 1;
921}
922
923static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
924{
925 struct fm10k_intfc *interface = netdev_priv(netdev);
926 struct fm10k_hw *hw = &interface->hw;
927 s32 err;
928 int i;
929
930
931 if (!vid)
932 return 0;
933
934 if (vid >= VLAN_N_VID)
935 return -EINVAL;
936
937
938 if (hw->mac.vlan_override)
939 return -EACCES;
940
941
942 set_bit(vid, interface->active_vlans);
943 if (!set)
944 clear_bit(vid, interface->active_vlans);
945
946
947 for (i = 0; i < interface->num_rx_queues; i++) {
948 struct fm10k_ring *rx_ring = interface->rx_ring[i];
949 u16 rx_vid = rx_ring->vid & (VLAN_N_VID - 1);
950
951 if (test_bit(rx_vid, interface->active_vlans))
952 rx_ring->vid |= FM10K_VLAN_CLEAR;
953 else
954 rx_ring->vid &= ~FM10K_VLAN_CLEAR;
955 }
956
957
958
959
960 if (!set && vid == hw->mac.default_vid)
961 return 0;
962
963
964
965
966 if (test_bit(__FM10K_DOWN, interface->state))
967 return 0;
968
969 fm10k_mbx_lock(interface);
970
971
972 if (!(netdev->flags & IFF_PROMISC)) {
973 err = fm10k_queue_vlan_request(interface, vid, 0, set);
974 if (err)
975 goto err_out;
976 }
977
978
979 err = fm10k_queue_mac_request(interface, interface->glort,
980 hw->mac.addr, vid, set);
981 if (err)
982 goto err_out;
983
984
985 interface->vid = vid + (set ? VLAN_N_VID : 0);
986
987
988 __dev_uc_unsync(netdev, fm10k_uc_vlan_unsync);
989 __dev_mc_unsync(netdev, fm10k_mc_vlan_unsync);
990
991err_out:
992 fm10k_mbx_unlock(interface);
993
994 return err;
995}
996
997static int fm10k_vlan_rx_add_vid(struct net_device *netdev,
998 __always_unused __be16 proto, u16 vid)
999{
1000
1001 return fm10k_update_vid(netdev, vid, true);
1002}
1003
1004static int fm10k_vlan_rx_kill_vid(struct net_device *netdev,
1005 __always_unused __be16 proto, u16 vid)
1006{
1007
1008 return fm10k_update_vid(netdev, vid, false);
1009}
1010
1011static u16 fm10k_find_next_vlan(struct fm10k_intfc *interface, u16 vid)
1012{
1013 struct fm10k_hw *hw = &interface->hw;
1014 u16 default_vid = hw->mac.default_vid;
1015 u16 vid_limit = vid < default_vid ? default_vid : VLAN_N_VID;
1016
1017 vid = find_next_bit(interface->active_vlans, vid_limit, ++vid);
1018
1019 return vid;
1020}
1021
1022static void fm10k_clear_unused_vlans(struct fm10k_intfc *interface)
1023{
1024 u32 vid, prev_vid;
1025
1026
1027 for (vid = 0, prev_vid = 0;
1028 prev_vid < VLAN_N_VID;
1029 prev_vid = vid + 1, vid = fm10k_find_next_vlan(interface, vid)) {
1030 if (prev_vid == vid)
1031 continue;
1032
1033
1034 prev_vid += (vid - prev_vid - 1) << FM10K_VLAN_LENGTH_SHIFT;
1035 fm10k_queue_vlan_request(interface, prev_vid, 0, false);
1036 }
1037}
1038
1039static int __fm10k_uc_sync(struct net_device *dev,
1040 const unsigned char *addr, bool sync)
1041{
1042 struct fm10k_intfc *interface = netdev_priv(dev);
1043 struct fm10k_hw *hw = &interface->hw;
1044 u16 vid, glort = interface->glort;
1045 s32 err;
1046
1047 if (!is_valid_ether_addr(addr))
1048 return -EADDRNOTAVAIL;
1049
1050 for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
1051 vid < VLAN_N_VID;
1052 vid = fm10k_find_next_vlan(interface, vid)) {
1053 err = fm10k_queue_mac_request(interface, glort,
1054 addr, vid, sync);
1055 if (err)
1056 return err;
1057 }
1058
1059 return 0;
1060}
1061
1062static int fm10k_uc_sync(struct net_device *dev,
1063 const unsigned char *addr)
1064{
1065 return __fm10k_uc_sync(dev, addr, true);
1066}
1067
1068static int fm10k_uc_unsync(struct net_device *dev,
1069 const unsigned char *addr)
1070{
1071 return __fm10k_uc_sync(dev, addr, false);
1072}
1073
1074static int fm10k_set_mac(struct net_device *dev, void *p)
1075{
1076 struct fm10k_intfc *interface = netdev_priv(dev);
1077 struct fm10k_hw *hw = &interface->hw;
1078 struct sockaddr *addr = p;
1079 s32 err = 0;
1080
1081 if (!is_valid_ether_addr(addr->sa_data))
1082 return -EADDRNOTAVAIL;
1083
1084 if (dev->flags & IFF_UP) {
1085
1086 fm10k_mbx_lock(interface);
1087
1088 err = fm10k_uc_sync(dev, addr->sa_data);
1089 if (!err)
1090 fm10k_uc_unsync(dev, hw->mac.addr);
1091
1092 fm10k_mbx_unlock(interface);
1093 }
1094
1095 if (!err) {
1096 ether_addr_copy(dev->dev_addr, addr->sa_data);
1097 ether_addr_copy(hw->mac.addr, addr->sa_data);
1098 dev->addr_assign_type &= ~NET_ADDR_RANDOM;
1099 }
1100
1101
1102 return err ? -EAGAIN : 0;
1103}
1104
1105static int __fm10k_mc_sync(struct net_device *dev,
1106 const unsigned char *addr, bool sync)
1107{
1108 struct fm10k_intfc *interface = netdev_priv(dev);
1109 struct fm10k_hw *hw = &interface->hw;
1110 u16 vid, glort = interface->glort;
1111 s32 err;
1112
1113 if (!is_multicast_ether_addr(addr))
1114 return -EADDRNOTAVAIL;
1115
1116 for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
1117 vid < VLAN_N_VID;
1118 vid = fm10k_find_next_vlan(interface, vid)) {
1119 err = fm10k_queue_mac_request(interface, glort,
1120 addr, vid, sync);
1121 if (err)
1122 return err;
1123 }
1124
1125 return 0;
1126}
1127
1128static int fm10k_mc_sync(struct net_device *dev,
1129 const unsigned char *addr)
1130{
1131 return __fm10k_mc_sync(dev, addr, true);
1132}
1133
1134static int fm10k_mc_unsync(struct net_device *dev,
1135 const unsigned char *addr)
1136{
1137 return __fm10k_mc_sync(dev, addr, false);
1138}
1139
1140static void fm10k_set_rx_mode(struct net_device *dev)
1141{
1142 struct fm10k_intfc *interface = netdev_priv(dev);
1143 struct fm10k_hw *hw = &interface->hw;
1144 int xcast_mode;
1145
1146
1147 if (!(dev->flags & IFF_UP))
1148 return;
1149
1150
1151 xcast_mode = (dev->flags & IFF_PROMISC) ? FM10K_XCAST_MODE_PROMISC :
1152 (dev->flags & IFF_ALLMULTI) ? FM10K_XCAST_MODE_ALLMULTI :
1153 (dev->flags & (IFF_BROADCAST | IFF_MULTICAST)) ?
1154 FM10K_XCAST_MODE_MULTI : FM10K_XCAST_MODE_NONE;
1155
1156 fm10k_mbx_lock(interface);
1157
1158
1159 if (interface->xcast_mode != xcast_mode) {
1160
1161 if (xcast_mode == FM10K_XCAST_MODE_PROMISC)
1162 fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL,
1163 0, true);
1164 if (interface->xcast_mode == FM10K_XCAST_MODE_PROMISC)
1165 fm10k_clear_unused_vlans(interface);
1166
1167
1168 if (fm10k_host_mbx_ready(interface))
1169 hw->mac.ops.update_xcast_mode(hw, interface->glort,
1170 xcast_mode);
1171
1172
1173 interface->xcast_mode = xcast_mode;
1174 }
1175
1176
1177 __dev_uc_sync(dev, fm10k_uc_sync, fm10k_uc_unsync);
1178 __dev_mc_sync(dev, fm10k_mc_sync, fm10k_mc_unsync);
1179
1180 fm10k_mbx_unlock(interface);
1181}
1182
1183void fm10k_restore_rx_state(struct fm10k_intfc *interface)
1184{
1185 struct net_device *netdev = interface->netdev;
1186 struct fm10k_hw *hw = &interface->hw;
1187 int xcast_mode;
1188 u16 vid, glort;
1189
1190
1191 glort = interface->glort;
1192
1193
1194 if (netdev->flags & IFF_PROMISC)
1195 xcast_mode = FM10K_XCAST_MODE_PROMISC;
1196 else if (netdev->flags & IFF_ALLMULTI)
1197 xcast_mode = FM10K_XCAST_MODE_ALLMULTI;
1198 else if (netdev->flags & (IFF_BROADCAST | IFF_MULTICAST))
1199 xcast_mode = FM10K_XCAST_MODE_MULTI;
1200 else
1201 xcast_mode = FM10K_XCAST_MODE_NONE;
1202
1203 fm10k_mbx_lock(interface);
1204
1205
1206 if (fm10k_host_mbx_ready(interface))
1207 hw->mac.ops.update_lport_state(hw, glort,
1208 interface->glort_count, true);
1209
1210
1211 fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL, 0,
1212 xcast_mode == FM10K_XCAST_MODE_PROMISC);
1213
1214
1215 fm10k_queue_vlan_request(interface, 0, 0, true);
1216
1217
1218 for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
1219 vid < VLAN_N_VID;
1220 vid = fm10k_find_next_vlan(interface, vid)) {
1221 fm10k_queue_vlan_request(interface, vid, 0, true);
1222
1223 fm10k_queue_mac_request(interface, glort,
1224 hw->mac.addr, vid, true);
1225 }
1226
1227
1228
1229
1230 if (fm10k_host_mbx_ready(interface))
1231 hw->mac.ops.update_xcast_mode(hw, glort, xcast_mode);
1232
1233
1234 __dev_uc_sync(netdev, fm10k_uc_sync, fm10k_uc_unsync);
1235 __dev_mc_sync(netdev, fm10k_mc_sync, fm10k_mc_unsync);
1236
1237 fm10k_mbx_unlock(interface);
1238
1239
1240 interface->xcast_mode = xcast_mode;
1241
1242
1243 fm10k_restore_udp_port_info(interface);
1244}
1245
1246void fm10k_reset_rx_state(struct fm10k_intfc *interface)
1247{
1248 struct net_device *netdev = interface->netdev;
1249 struct fm10k_hw *hw = &interface->hw;
1250
1251
1252 while (test_bit(__FM10K_MACVLAN_SCHED, interface->state))
1253 usleep_range(1000, 2000);
1254
1255
1256 fm10k_clear_macvlan_queue(interface, interface->glort, true);
1257
1258 fm10k_mbx_lock(interface);
1259
1260
1261
1262
1263 if (fm10k_host_mbx_ready(interface))
1264 hw->mac.ops.update_lport_state(hw, interface->glort,
1265 interface->glort_count, false);
1266
1267 fm10k_mbx_unlock(interface);
1268
1269
1270 interface->xcast_mode = FM10K_XCAST_MODE_NONE;
1271
1272
1273 __dev_uc_unsync(netdev, NULL);
1274 __dev_mc_unsync(netdev, NULL);
1275}
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285static void fm10k_get_stats64(struct net_device *netdev,
1286 struct rtnl_link_stats64 *stats)
1287{
1288 struct fm10k_intfc *interface = netdev_priv(netdev);
1289 struct fm10k_ring *ring;
1290 unsigned int start, i;
1291 u64 bytes, packets;
1292
1293 rcu_read_lock();
1294
1295 for (i = 0; i < interface->num_rx_queues; i++) {
1296 ring = READ_ONCE(interface->rx_ring[i]);
1297
1298 if (!ring)
1299 continue;
1300
1301 do {
1302 start = u64_stats_fetch_begin_irq(&ring->syncp);
1303 packets = ring->stats.packets;
1304 bytes = ring->stats.bytes;
1305 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1306
1307 stats->rx_packets += packets;
1308 stats->rx_bytes += bytes;
1309 }
1310
1311 for (i = 0; i < interface->num_tx_queues; i++) {
1312 ring = READ_ONCE(interface->tx_ring[i]);
1313
1314 if (!ring)
1315 continue;
1316
1317 do {
1318 start = u64_stats_fetch_begin_irq(&ring->syncp);
1319 packets = ring->stats.packets;
1320 bytes = ring->stats.bytes;
1321 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1322
1323 stats->tx_packets += packets;
1324 stats->tx_bytes += bytes;
1325 }
1326
1327 rcu_read_unlock();
1328
1329
1330 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1331}
1332
1333int fm10k_setup_tc(struct net_device *dev, u8 tc)
1334{
1335 struct fm10k_intfc *interface = netdev_priv(dev);
1336 int err;
1337
1338
1339 if (tc && (interface->hw.mac.type != fm10k_mac_pf))
1340 return -EINVAL;
1341
1342
1343 if (tc > 8)
1344 return -EINVAL;
1345
1346
1347
1348
1349
1350 if (netif_running(dev))
1351 fm10k_close(dev);
1352
1353 fm10k_mbx_free_irq(interface);
1354
1355 fm10k_clear_queueing_scheme(interface);
1356
1357
1358 netdev_reset_tc(dev);
1359 netdev_set_num_tc(dev, tc);
1360
1361 err = fm10k_init_queueing_scheme(interface);
1362 if (err)
1363 goto err_queueing_scheme;
1364
1365 err = fm10k_mbx_request_irq(interface);
1366 if (err)
1367 goto err_mbx_irq;
1368
1369 err = netif_running(dev) ? fm10k_open(dev) : 0;
1370 if (err)
1371 goto err_open;
1372
1373
1374 set_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags);
1375
1376 return 0;
1377err_open:
1378 fm10k_mbx_free_irq(interface);
1379err_mbx_irq:
1380 fm10k_clear_queueing_scheme(interface);
1381err_queueing_scheme:
1382 netif_device_detach(dev);
1383
1384 return err;
1385}
1386
1387static int __fm10k_setup_tc(struct net_device *dev, enum tc_setup_type type,
1388 void *type_data)
1389{
1390 struct tc_mqprio_qopt *mqprio = type_data;
1391
1392 if (type != TC_SETUP_QDISC_MQPRIO)
1393 return -EOPNOTSUPP;
1394
1395 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1396
1397 return fm10k_setup_tc(dev, mqprio->num_tc);
1398}
1399
1400static void fm10k_assign_l2_accel(struct fm10k_intfc *interface,
1401 struct fm10k_l2_accel *l2_accel)
1402{
1403 struct fm10k_ring *ring;
1404 int i;
1405
1406 for (i = 0; i < interface->num_rx_queues; i++) {
1407 ring = interface->rx_ring[i];
1408 rcu_assign_pointer(ring->l2_accel, l2_accel);
1409 }
1410
1411 interface->l2_accel = l2_accel;
1412}
1413
1414static void *fm10k_dfwd_add_station(struct net_device *dev,
1415 struct net_device *sdev)
1416{
1417 struct fm10k_intfc *interface = netdev_priv(dev);
1418 struct fm10k_l2_accel *l2_accel = interface->l2_accel;
1419 struct fm10k_l2_accel *old_l2_accel = NULL;
1420 struct fm10k_dglort_cfg dglort = { 0 };
1421 struct fm10k_hw *hw = &interface->hw;
1422 int size = 0, i;
1423 u16 glort;
1424
1425
1426 if (!l2_accel) {
1427
1428 if (interface->glort_count < 7)
1429 return ERR_PTR(-EBUSY);
1430
1431 size = offsetof(struct fm10k_l2_accel, macvlan[7]);
1432 l2_accel = kzalloc(size, GFP_KERNEL);
1433 if (!l2_accel)
1434 return ERR_PTR(-ENOMEM);
1435
1436 l2_accel->size = 7;
1437 l2_accel->dglort = interface->glort;
1438
1439
1440 fm10k_assign_l2_accel(interface, l2_accel);
1441
1442 } else if ((l2_accel->count == FM10K_MAX_STATIONS) ||
1443 (l2_accel->count == (interface->glort_count - 1))) {
1444 return ERR_PTR(-EBUSY);
1445
1446 } else if (l2_accel->count == l2_accel->size) {
1447 old_l2_accel = l2_accel;
1448 size = offsetof(struct fm10k_l2_accel,
1449 macvlan[(l2_accel->size * 2) + 1]);
1450 l2_accel = kzalloc(size, GFP_KERNEL);
1451 if (!l2_accel)
1452 return ERR_PTR(-ENOMEM);
1453
1454 memcpy(l2_accel, old_l2_accel,
1455 offsetof(struct fm10k_l2_accel,
1456 macvlan[old_l2_accel->size]));
1457
1458 l2_accel->size = (old_l2_accel->size * 2) + 1;
1459
1460
1461 fm10k_assign_l2_accel(interface, l2_accel);
1462 kfree_rcu(old_l2_accel, rcu);
1463 }
1464
1465
1466 for (i = 0; i < l2_accel->size; i++) {
1467 if (!l2_accel->macvlan[i])
1468 break;
1469 }
1470
1471
1472 l2_accel->macvlan[i] = sdev;
1473 l2_accel->count++;
1474
1475
1476 dglort.idx = fm10k_dglort_pf_rss;
1477 dglort.inner_rss = 1;
1478 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1479 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1480 dglort.glort = interface->glort;
1481 dglort.shared_l = fls(l2_accel->size);
1482 hw->mac.ops.configure_dglort_map(hw, &dglort);
1483
1484
1485 fm10k_mbx_lock(interface);
1486
1487 glort = l2_accel->dglort + 1 + i;
1488
1489 if (fm10k_host_mbx_ready(interface)) {
1490 hw->mac.ops.update_xcast_mode(hw, glort,
1491 FM10K_XCAST_MODE_MULTI);
1492 fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
1493 0, true);
1494 }
1495
1496 fm10k_mbx_unlock(interface);
1497
1498 return sdev;
1499}
1500
1501static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
1502{
1503 struct fm10k_intfc *interface = netdev_priv(dev);
1504 struct fm10k_l2_accel *l2_accel = READ_ONCE(interface->l2_accel);
1505 struct fm10k_dglort_cfg dglort = { 0 };
1506 struct fm10k_hw *hw = &interface->hw;
1507 struct net_device *sdev = priv;
1508 int i;
1509 u16 glort;
1510
1511 if (!l2_accel)
1512 return;
1513
1514
1515 for (i = 0; i < l2_accel->size; i++) {
1516 if (l2_accel->macvlan[i] == sdev)
1517 break;
1518 }
1519
1520
1521 if (i == l2_accel->size)
1522 return;
1523
1524
1525 fm10k_mbx_lock(interface);
1526
1527 glort = l2_accel->dglort + 1 + i;
1528
1529 if (fm10k_host_mbx_ready(interface)) {
1530 hw->mac.ops.update_xcast_mode(hw, glort,
1531 FM10K_XCAST_MODE_NONE);
1532 fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
1533 0, false);
1534 }
1535
1536 fm10k_mbx_unlock(interface);
1537
1538
1539 l2_accel->macvlan[i] = NULL;
1540 l2_accel->count--;
1541
1542
1543 dglort.idx = fm10k_dglort_pf_rss;
1544 dglort.inner_rss = 1;
1545 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1546 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1547 dglort.glort = interface->glort;
1548 dglort.shared_l = fls(l2_accel->size);
1549 hw->mac.ops.configure_dglort_map(hw, &dglort);
1550
1551
1552 if (l2_accel->count == 0) {
1553 fm10k_assign_l2_accel(interface, NULL);
1554 kfree_rcu(l2_accel, rcu);
1555 }
1556}
1557
1558static netdev_features_t fm10k_features_check(struct sk_buff *skb,
1559 struct net_device *dev,
1560 netdev_features_t features)
1561{
1562 if (!skb->encapsulation || fm10k_tx_encap_offload(skb))
1563 return features;
1564
1565 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1566}
1567
1568static const struct net_device_ops fm10k_netdev_ops = {
1569 .ndo_open = fm10k_open,
1570 .ndo_stop = fm10k_close,
1571 .ndo_validate_addr = eth_validate_addr,
1572 .ndo_start_xmit = fm10k_xmit_frame,
1573 .ndo_set_mac_address = fm10k_set_mac,
1574 .ndo_tx_timeout = fm10k_tx_timeout,
1575 .ndo_vlan_rx_add_vid = fm10k_vlan_rx_add_vid,
1576 .ndo_vlan_rx_kill_vid = fm10k_vlan_rx_kill_vid,
1577 .ndo_set_rx_mode = fm10k_set_rx_mode,
1578 .ndo_get_stats64 = fm10k_get_stats64,
1579 .ndo_setup_tc = __fm10k_setup_tc,
1580 .ndo_set_vf_mac = fm10k_ndo_set_vf_mac,
1581 .ndo_set_vf_vlan = fm10k_ndo_set_vf_vlan,
1582 .ndo_set_vf_rate = fm10k_ndo_set_vf_bw,
1583 .ndo_get_vf_config = fm10k_ndo_get_vf_config,
1584 .ndo_udp_tunnel_add = fm10k_udp_tunnel_add,
1585 .ndo_udp_tunnel_del = fm10k_udp_tunnel_del,
1586 .ndo_dfwd_add_station = fm10k_dfwd_add_station,
1587 .ndo_dfwd_del_station = fm10k_dfwd_del_station,
1588#ifdef CONFIG_NET_POLL_CONTROLLER
1589 .ndo_poll_controller = fm10k_netpoll,
1590#endif
1591 .ndo_features_check = fm10k_features_check,
1592};
1593
1594#define DEFAULT_DEBUG_LEVEL_SHIFT 3
1595
1596struct net_device *fm10k_alloc_netdev(const struct fm10k_info *info)
1597{
1598 netdev_features_t hw_features;
1599 struct fm10k_intfc *interface;
1600 struct net_device *dev;
1601
1602 dev = alloc_etherdev_mq(sizeof(struct fm10k_intfc), MAX_QUEUES);
1603 if (!dev)
1604 return NULL;
1605
1606
1607 dev->netdev_ops = &fm10k_netdev_ops;
1608 fm10k_set_ethtool_ops(dev);
1609
1610
1611 interface = netdev_priv(dev);
1612 interface->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
1613
1614
1615 dev->features |= NETIF_F_IP_CSUM |
1616 NETIF_F_IPV6_CSUM |
1617 NETIF_F_SG |
1618 NETIF_F_TSO |
1619 NETIF_F_TSO6 |
1620 NETIF_F_TSO_ECN |
1621 NETIF_F_RXHASH |
1622 NETIF_F_RXCSUM;
1623
1624
1625 if (info->mac == fm10k_mac_pf) {
1626 dev->hw_enc_features = NETIF_F_IP_CSUM |
1627 NETIF_F_TSO |
1628 NETIF_F_TSO6 |
1629 NETIF_F_TSO_ECN |
1630 NETIF_F_GSO_UDP_TUNNEL |
1631 NETIF_F_IPV6_CSUM |
1632 NETIF_F_SG;
1633
1634 dev->features |= NETIF_F_GSO_UDP_TUNNEL;
1635 }
1636
1637
1638 hw_features = dev->features;
1639
1640
1641 hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
1642
1643
1644 dev->vlan_features |= dev->features;
1645
1646
1647
1648
1649
1650 dev->features |= NETIF_F_HW_VLAN_CTAG_TX |
1651 NETIF_F_HW_VLAN_CTAG_RX |
1652 NETIF_F_HW_VLAN_CTAG_FILTER;
1653
1654 dev->priv_flags |= IFF_UNICAST_FLT;
1655
1656 dev->hw_features |= hw_features;
1657
1658
1659 dev->min_mtu = ETH_MIN_MTU;
1660 dev->max_mtu = FM10K_MAX_JUMBO_FRAME_SIZE;
1661
1662 return dev;
1663}
1664