1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include "fm10k.h"
23#include <linux/vmalloc.h>
24#include <net/udp_tunnel.h>
25
26
27
28
29
30
31
32int fm10k_setup_tx_resources(struct fm10k_ring *tx_ring)
33{
34 struct device *dev = tx_ring->dev;
35 int size;
36
37 size = sizeof(struct fm10k_tx_buffer) * tx_ring->count;
38
39 tx_ring->tx_buffer = vzalloc(size);
40 if (!tx_ring->tx_buffer)
41 goto err;
42
43 u64_stats_init(&tx_ring->syncp);
44
45
46 tx_ring->size = tx_ring->count * sizeof(struct fm10k_tx_desc);
47 tx_ring->size = ALIGN(tx_ring->size, 4096);
48
49 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
50 &tx_ring->dma, GFP_KERNEL);
51 if (!tx_ring->desc)
52 goto err;
53
54 return 0;
55
56err:
57 vfree(tx_ring->tx_buffer);
58 tx_ring->tx_buffer = NULL;
59 return -ENOMEM;
60}
61
62
63
64
65
66
67
68
69
70
71
72static int fm10k_setup_all_tx_resources(struct fm10k_intfc *interface)
73{
74 int i, err = 0;
75
76 for (i = 0; i < interface->num_tx_queues; i++) {
77 err = fm10k_setup_tx_resources(interface->tx_ring[i]);
78 if (!err)
79 continue;
80
81 netif_err(interface, probe, interface->netdev,
82 "Allocation for Tx Queue %u failed\n", i);
83 goto err_setup_tx;
84 }
85
86 return 0;
87err_setup_tx:
88
89 while (i--)
90 fm10k_free_tx_resources(interface->tx_ring[i]);
91 return err;
92}
93
94
95
96
97
98
99
100int fm10k_setup_rx_resources(struct fm10k_ring *rx_ring)
101{
102 struct device *dev = rx_ring->dev;
103 int size;
104
105 size = sizeof(struct fm10k_rx_buffer) * rx_ring->count;
106
107 rx_ring->rx_buffer = vzalloc(size);
108 if (!rx_ring->rx_buffer)
109 goto err;
110
111 u64_stats_init(&rx_ring->syncp);
112
113
114 rx_ring->size = rx_ring->count * sizeof(union fm10k_rx_desc);
115 rx_ring->size = ALIGN(rx_ring->size, 4096);
116
117 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
118 &rx_ring->dma, GFP_KERNEL);
119 if (!rx_ring->desc)
120 goto err;
121
122 return 0;
123err:
124 vfree(rx_ring->rx_buffer);
125 rx_ring->rx_buffer = NULL;
126 return -ENOMEM;
127}
128
129
130
131
132
133
134
135
136
137
138
139static int fm10k_setup_all_rx_resources(struct fm10k_intfc *interface)
140{
141 int i, err = 0;
142
143 for (i = 0; i < interface->num_rx_queues; i++) {
144 err = fm10k_setup_rx_resources(interface->rx_ring[i]);
145 if (!err)
146 continue;
147
148 netif_err(interface, probe, interface->netdev,
149 "Allocation for Rx Queue %u failed\n", i);
150 goto err_setup_rx;
151 }
152
153 return 0;
154err_setup_rx:
155
156 while (i--)
157 fm10k_free_rx_resources(interface->rx_ring[i]);
158 return err;
159}
160
161void fm10k_unmap_and_free_tx_resource(struct fm10k_ring *ring,
162 struct fm10k_tx_buffer *tx_buffer)
163{
164 if (tx_buffer->skb) {
165 dev_kfree_skb_any(tx_buffer->skb);
166 if (dma_unmap_len(tx_buffer, len))
167 dma_unmap_single(ring->dev,
168 dma_unmap_addr(tx_buffer, dma),
169 dma_unmap_len(tx_buffer, len),
170 DMA_TO_DEVICE);
171 } else if (dma_unmap_len(tx_buffer, len)) {
172 dma_unmap_page(ring->dev,
173 dma_unmap_addr(tx_buffer, dma),
174 dma_unmap_len(tx_buffer, len),
175 DMA_TO_DEVICE);
176 }
177 tx_buffer->next_to_watch = NULL;
178 tx_buffer->skb = NULL;
179 dma_unmap_len_set(tx_buffer, len, 0);
180
181}
182
183
184
185
186
187static void fm10k_clean_tx_ring(struct fm10k_ring *tx_ring)
188{
189 struct fm10k_tx_buffer *tx_buffer;
190 unsigned long size;
191 u16 i;
192
193
194 if (!tx_ring->tx_buffer)
195 return;
196
197
198 for (i = 0; i < tx_ring->count; i++) {
199 tx_buffer = &tx_ring->tx_buffer[i];
200 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer);
201 }
202
203
204 netdev_tx_reset_queue(txring_txq(tx_ring));
205
206 size = sizeof(struct fm10k_tx_buffer) * tx_ring->count;
207 memset(tx_ring->tx_buffer, 0, size);
208
209
210 memset(tx_ring->desc, 0, tx_ring->size);
211}
212
213
214
215
216
217
218
219void fm10k_free_tx_resources(struct fm10k_ring *tx_ring)
220{
221 fm10k_clean_tx_ring(tx_ring);
222
223 vfree(tx_ring->tx_buffer);
224 tx_ring->tx_buffer = NULL;
225
226
227 if (!tx_ring->desc)
228 return;
229
230 dma_free_coherent(tx_ring->dev, tx_ring->size,
231 tx_ring->desc, tx_ring->dma);
232 tx_ring->desc = NULL;
233}
234
235
236
237
238
239void fm10k_clean_all_tx_rings(struct fm10k_intfc *interface)
240{
241 int i;
242
243 for (i = 0; i < interface->num_tx_queues; i++)
244 fm10k_clean_tx_ring(interface->tx_ring[i]);
245}
246
247
248
249
250
251
252
253static void fm10k_free_all_tx_resources(struct fm10k_intfc *interface)
254{
255 int i = interface->num_tx_queues;
256
257 while (i--)
258 fm10k_free_tx_resources(interface->tx_ring[i]);
259}
260
261
262
263
264
265static void fm10k_clean_rx_ring(struct fm10k_ring *rx_ring)
266{
267 unsigned long size;
268 u16 i;
269
270 if (!rx_ring->rx_buffer)
271 return;
272
273 if (rx_ring->skb)
274 dev_kfree_skb(rx_ring->skb);
275 rx_ring->skb = NULL;
276
277
278 for (i = 0; i < rx_ring->count; i++) {
279 struct fm10k_rx_buffer *buffer = &rx_ring->rx_buffer[i];
280
281 if (!buffer->page)
282 continue;
283
284 dma_unmap_page(rx_ring->dev, buffer->dma,
285 PAGE_SIZE, DMA_FROM_DEVICE);
286 __free_page(buffer->page);
287
288 buffer->page = NULL;
289 }
290
291 size = sizeof(struct fm10k_rx_buffer) * rx_ring->count;
292 memset(rx_ring->rx_buffer, 0, size);
293
294
295 memset(rx_ring->desc, 0, rx_ring->size);
296
297 rx_ring->next_to_alloc = 0;
298 rx_ring->next_to_clean = 0;
299 rx_ring->next_to_use = 0;
300}
301
302
303
304
305
306
307
308void fm10k_free_rx_resources(struct fm10k_ring *rx_ring)
309{
310 fm10k_clean_rx_ring(rx_ring);
311
312 vfree(rx_ring->rx_buffer);
313 rx_ring->rx_buffer = NULL;
314
315
316 if (!rx_ring->desc)
317 return;
318
319 dma_free_coherent(rx_ring->dev, rx_ring->size,
320 rx_ring->desc, rx_ring->dma);
321
322 rx_ring->desc = NULL;
323}
324
325
326
327
328
329void fm10k_clean_all_rx_rings(struct fm10k_intfc *interface)
330{
331 int i;
332
333 for (i = 0; i < interface->num_rx_queues; i++)
334 fm10k_clean_rx_ring(interface->rx_ring[i]);
335}
336
337
338
339
340
341
342
343static void fm10k_free_all_rx_resources(struct fm10k_intfc *interface)
344{
345 int i = interface->num_rx_queues;
346
347 while (i--)
348 fm10k_free_rx_resources(interface->rx_ring[i]);
349}
350
351
352
353
354
355
356
357static void fm10k_request_glort_range(struct fm10k_intfc *interface)
358{
359 struct fm10k_hw *hw = &interface->hw;
360 u16 mask = (~hw->mac.dglort_map) >> FM10K_DGLORTMAP_MASK_SHIFT;
361
362
363 interface->glort = hw->mac.dglort_map & FM10K_DGLORTMAP_NONE;
364 interface->glort_count = 0;
365
366
367 if (hw->mac.dglort_map == FM10K_DGLORTMAP_NONE)
368 return;
369
370
371
372
373
374
375 if (mask <= hw->iov.total_vfs) {
376 interface->glort_count = 1;
377 interface->glort += mask;
378 } else if (mask < 64) {
379 interface->glort_count = (mask + 1) / 2;
380 interface->glort += interface->glort_count;
381 } else {
382 interface->glort_count = mask - 63;
383 interface->glort += 64;
384 }
385}
386
387
388
389
390
391
392
393static void fm10k_free_udp_port_info(struct fm10k_intfc *interface)
394{
395 struct fm10k_udp_port *port;
396
397
398 port = list_first_entry_or_null(&interface->vxlan_port,
399 struct fm10k_udp_port, list);
400 while (port) {
401 list_del(&port->list);
402 kfree(port);
403 port = list_first_entry_or_null(&interface->vxlan_port,
404 struct fm10k_udp_port,
405 list);
406 }
407
408
409 port = list_first_entry_or_null(&interface->geneve_port,
410 struct fm10k_udp_port, list);
411 while (port) {
412 list_del(&port->list);
413 kfree(port);
414 port = list_first_entry_or_null(&interface->vxlan_port,
415 struct fm10k_udp_port,
416 list);
417 }
418}
419
420
421
422
423
424
425
426static void fm10k_restore_udp_port_info(struct fm10k_intfc *interface)
427{
428 struct fm10k_hw *hw = &interface->hw;
429 struct fm10k_udp_port *port;
430
431
432 if (hw->mac.type != fm10k_mac_pf)
433 return;
434
435 port = list_first_entry_or_null(&interface->vxlan_port,
436 struct fm10k_udp_port, list);
437
438
439 fm10k_write_reg(hw, FM10K_TUNNEL_CFG,
440 (port ? ntohs(port->port) : 0) |
441 (ETH_P_TEB << FM10K_TUNNEL_CFG_NVGRE_SHIFT));
442
443 port = list_first_entry_or_null(&interface->geneve_port,
444 struct fm10k_udp_port, list);
445
446
447 fm10k_write_reg(hw, FM10K_TUNNEL_CFG_GENEVE,
448 (port ? ntohs(port->port) : 0));
449}
450
451static struct fm10k_udp_port *
452fm10k_remove_tunnel_port(struct list_head *ports,
453 struct udp_tunnel_info *ti)
454{
455 struct fm10k_udp_port *port;
456
457 list_for_each_entry(port, ports, list) {
458 if ((port->port == ti->port) &&
459 (port->sa_family == ti->sa_family)) {
460 list_del(&port->list);
461 return port;
462 }
463 }
464
465 return NULL;
466}
467
468static void fm10k_insert_tunnel_port(struct list_head *ports,
469 struct udp_tunnel_info *ti)
470{
471 struct fm10k_udp_port *port;
472
473
474
475
476 port = fm10k_remove_tunnel_port(ports, ti);
477 if (!port) {
478 port = kmalloc(sizeof(*port), GFP_ATOMIC);
479 if (!port)
480 return;
481 port->port = ti->port;
482 port->sa_family = ti->sa_family;
483 }
484
485 list_add_tail(&port->list, ports);
486}
487
488
489
490
491
492
493
494
495
496
497static void fm10k_udp_tunnel_add(struct net_device *dev,
498 struct udp_tunnel_info *ti)
499{
500 struct fm10k_intfc *interface = netdev_priv(dev);
501
502
503 if (interface->hw.mac.type != fm10k_mac_pf)
504 return;
505
506 switch (ti->type) {
507 case UDP_TUNNEL_TYPE_VXLAN:
508 fm10k_insert_tunnel_port(&interface->vxlan_port, ti);
509 break;
510 case UDP_TUNNEL_TYPE_GENEVE:
511 fm10k_insert_tunnel_port(&interface->geneve_port, ti);
512 break;
513 default:
514 return;
515 }
516
517 fm10k_restore_udp_port_info(interface);
518}
519
520
521
522
523
524
525
526
527
528
529static void fm10k_udp_tunnel_del(struct net_device *dev,
530 struct udp_tunnel_info *ti)
531{
532 struct fm10k_intfc *interface = netdev_priv(dev);
533 struct fm10k_udp_port *port = NULL;
534
535 if (interface->hw.mac.type != fm10k_mac_pf)
536 return;
537
538 switch (ti->type) {
539 case UDP_TUNNEL_TYPE_VXLAN:
540 port = fm10k_remove_tunnel_port(&interface->vxlan_port, ti);
541 break;
542 case UDP_TUNNEL_TYPE_GENEVE:
543 port = fm10k_remove_tunnel_port(&interface->geneve_port, ti);
544 break;
545 default:
546 return;
547 }
548
549
550 kfree(port);
551
552 fm10k_restore_udp_port_info(interface);
553}
554
555
556
557
558
559
560
561
562
563
564
565
566
567int fm10k_open(struct net_device *netdev)
568{
569 struct fm10k_intfc *interface = netdev_priv(netdev);
570 int err;
571
572
573 err = fm10k_setup_all_tx_resources(interface);
574 if (err)
575 goto err_setup_tx;
576
577
578 err = fm10k_setup_all_rx_resources(interface);
579 if (err)
580 goto err_setup_rx;
581
582
583 err = fm10k_qv_request_irq(interface);
584 if (err)
585 goto err_req_irq;
586
587
588 fm10k_request_glort_range(interface);
589
590
591 err = netif_set_real_num_tx_queues(netdev,
592 interface->num_tx_queues);
593 if (err)
594 goto err_set_queues;
595
596 err = netif_set_real_num_rx_queues(netdev,
597 interface->num_rx_queues);
598 if (err)
599 goto err_set_queues;
600
601 udp_tunnel_get_rx_info(netdev);
602
603 fm10k_up(interface);
604
605 return 0;
606
607err_set_queues:
608 fm10k_qv_free_irq(interface);
609err_req_irq:
610 fm10k_free_all_rx_resources(interface);
611err_setup_rx:
612 fm10k_free_all_tx_resources(interface);
613err_setup_tx:
614 return err;
615}
616
617
618
619
620
621
622
623
624
625
626
627
628int fm10k_close(struct net_device *netdev)
629{
630 struct fm10k_intfc *interface = netdev_priv(netdev);
631
632 fm10k_down(interface);
633
634 fm10k_qv_free_irq(interface);
635
636 fm10k_free_udp_port_info(interface);
637
638 fm10k_free_all_tx_resources(interface);
639 fm10k_free_all_rx_resources(interface);
640
641 return 0;
642}
643
644static netdev_tx_t fm10k_xmit_frame(struct sk_buff *skb, struct net_device *dev)
645{
646 struct fm10k_intfc *interface = netdev_priv(dev);
647 int num_tx_queues = READ_ONCE(interface->num_tx_queues);
648 unsigned int r_idx = skb->queue_mapping;
649 int err;
650
651 if (!num_tx_queues)
652 return NETDEV_TX_BUSY;
653
654 if ((skb->protocol == htons(ETH_P_8021Q)) &&
655 !skb_vlan_tag_present(skb)) {
656
657
658
659 struct vlan_hdr *vhdr;
660 __be16 proto;
661
662
663 skb = skb_share_check(skb, GFP_ATOMIC);
664 if (!skb)
665 return NETDEV_TX_OK;
666
667
668 if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN)))
669 return NETDEV_TX_OK;
670
671
672 err = skb_cow_head(skb, 0);
673 if (err) {
674 dev_kfree_skb(skb);
675 return NETDEV_TX_OK;
676 }
677
678
679 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
680
681
682 __vlan_hwaccel_put_tag(skb,
683 htons(ETH_P_8021Q),
684 ntohs(vhdr->h_vlan_TCI));
685 proto = vhdr->h_vlan_encapsulated_proto;
686 skb->protocol = (ntohs(proto) >= 1536) ? proto :
687 htons(ETH_P_802_2);
688
689
690 memmove(skb->data + VLAN_HLEN, skb->data, 12);
691 __skb_pull(skb, VLAN_HLEN);
692 skb_reset_mac_header(skb);
693 }
694
695
696
697
698 if (unlikely(skb->len < 17)) {
699 int pad_len = 17 - skb->len;
700
701 if (skb_pad(skb, pad_len))
702 return NETDEV_TX_OK;
703 __skb_put(skb, pad_len);
704 }
705
706 if (r_idx >= num_tx_queues)
707 r_idx %= num_tx_queues;
708
709 err = fm10k_xmit_frame_ring(skb, interface->tx_ring[r_idx]);
710
711 return err;
712}
713
714
715
716
717
718static void fm10k_tx_timeout(struct net_device *netdev)
719{
720 struct fm10k_intfc *interface = netdev_priv(netdev);
721 bool real_tx_hang = false;
722 int i;
723
724#define TX_TIMEO_LIMIT 16000
725 for (i = 0; i < interface->num_tx_queues; i++) {
726 struct fm10k_ring *tx_ring = interface->tx_ring[i];
727
728 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring))
729 real_tx_hang = true;
730 }
731
732 if (real_tx_hang) {
733 fm10k_tx_timeout_reset(interface);
734 } else {
735 netif_info(interface, drv, netdev,
736 "Fake Tx hang detected with timeout of %d seconds\n",
737 netdev->watchdog_timeo / HZ);
738
739
740 if (netdev->watchdog_timeo < TX_TIMEO_LIMIT)
741 netdev->watchdog_timeo *= 2;
742 }
743}
744
745
746
747
748
749
750
751
752
753
754
755static bool fm10k_host_mbx_ready(struct fm10k_intfc *interface)
756{
757 struct fm10k_hw *hw = &interface->hw;
758
759 return (hw->mac.type == fm10k_mac_vf || interface->host_ready);
760}
761
762
763
764
765
766
767
768
769
770
771
772
773
774int fm10k_queue_vlan_request(struct fm10k_intfc *interface,
775 u32 vid, u8 vsi, bool set)
776{
777 struct fm10k_macvlan_request *request;
778 unsigned long flags;
779
780
781
782
783 request = kzalloc(sizeof(*request), GFP_ATOMIC);
784 if (!request)
785 return -ENOMEM;
786
787 request->type = FM10K_VLAN_REQUEST;
788 request->vlan.vid = vid;
789 request->vlan.vsi = vsi;
790 request->set = set;
791
792 spin_lock_irqsave(&interface->macvlan_lock, flags);
793 list_add_tail(&request->list, &interface->macvlan_requests);
794 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
795
796 fm10k_macvlan_schedule(interface);
797
798 return 0;
799}
800
801
802
803
804
805
806
807
808
809
810
811
812
813int fm10k_queue_mac_request(struct fm10k_intfc *interface, u16 glort,
814 const unsigned char *addr, u16 vid, bool set)
815{
816 struct fm10k_macvlan_request *request;
817 unsigned long flags;
818
819
820
821
822 request = kzalloc(sizeof(*request), GFP_ATOMIC);
823 if (!request)
824 return -ENOMEM;
825
826 if (is_multicast_ether_addr(addr))
827 request->type = FM10K_MC_MAC_REQUEST;
828 else
829 request->type = FM10K_UC_MAC_REQUEST;
830
831 ether_addr_copy(request->mac.addr, addr);
832 request->mac.glort = glort;
833 request->mac.vid = vid;
834 request->set = set;
835
836 spin_lock_irqsave(&interface->macvlan_lock, flags);
837 list_add_tail(&request->list, &interface->macvlan_requests);
838 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
839
840 fm10k_macvlan_schedule(interface);
841
842 return 0;
843}
844
845
846
847
848
849
850
851
852
853
854void fm10k_clear_macvlan_queue(struct fm10k_intfc *interface,
855 u16 glort, bool vlans)
856
857{
858 struct fm10k_macvlan_request *r, *tmp;
859 unsigned long flags;
860
861 spin_lock_irqsave(&interface->macvlan_lock, flags);
862
863
864 list_for_each_entry_safe(r, tmp, &interface->macvlan_requests, list) {
865 switch (r->type) {
866 case FM10K_MC_MAC_REQUEST:
867 case FM10K_UC_MAC_REQUEST:
868
869 if (r->mac.glort != glort)
870 break;
871
872 case FM10K_VLAN_REQUEST:
873 if (vlans) {
874 list_del(&r->list);
875 kfree(r);
876 }
877 break;
878 }
879 }
880
881 spin_unlock_irqrestore(&interface->macvlan_lock, flags);
882}
883
884static int fm10k_uc_vlan_unsync(struct net_device *netdev,
885 const unsigned char *uc_addr)
886{
887 struct fm10k_intfc *interface = netdev_priv(netdev);
888 u16 glort = interface->glort;
889 u16 vid = interface->vid;
890 bool set = !!(vid / VLAN_N_VID);
891 int err = -EHOSTDOWN;
892
893
894 vid &= VLAN_N_VID - 1;
895
896 err = fm10k_queue_mac_request(interface, glort, uc_addr, vid, set);
897 if (err)
898 return err;
899
900
901 return 1;
902}
903
904static int fm10k_mc_vlan_unsync(struct net_device *netdev,
905 const unsigned char *mc_addr)
906{
907 struct fm10k_intfc *interface = netdev_priv(netdev);
908 u16 glort = interface->glort;
909 u16 vid = interface->vid;
910 bool set = !!(vid / VLAN_N_VID);
911 int err = -EHOSTDOWN;
912
913
914 vid &= VLAN_N_VID - 1;
915
916 err = fm10k_queue_mac_request(interface, glort, mc_addr, vid, set);
917 if (err)
918 return err;
919
920
921 return 1;
922}
923
924static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
925{
926 struct fm10k_intfc *interface = netdev_priv(netdev);
927 struct fm10k_hw *hw = &interface->hw;
928 s32 err;
929 int i;
930
931
932 if (!vid)
933 return 0;
934
935 if (vid >= VLAN_N_VID)
936 return -EINVAL;
937
938
939
940
941
942
943 if (set && hw->mac.vlan_override)
944 return -EACCES;
945
946
947 set_bit(vid, interface->active_vlans);
948 if (!set)
949 clear_bit(vid, interface->active_vlans);
950
951
952 for (i = 0; i < interface->num_rx_queues; i++) {
953 struct fm10k_ring *rx_ring = interface->rx_ring[i];
954 u16 rx_vid = rx_ring->vid & (VLAN_N_VID - 1);
955
956 if (test_bit(rx_vid, interface->active_vlans))
957 rx_ring->vid |= FM10K_VLAN_CLEAR;
958 else
959 rx_ring->vid &= ~FM10K_VLAN_CLEAR;
960 }
961
962
963
964
965 if (hw->mac.vlan_override)
966 return 0;
967
968
969
970
971 if (!set && vid == hw->mac.default_vid)
972 return 0;
973
974
975
976
977 if (test_bit(__FM10K_DOWN, interface->state))
978 return 0;
979
980 fm10k_mbx_lock(interface);
981
982
983 if (!(netdev->flags & IFF_PROMISC)) {
984 err = fm10k_queue_vlan_request(interface, vid, 0, set);
985 if (err)
986 goto err_out;
987 }
988
989
990 err = fm10k_queue_mac_request(interface, interface->glort,
991 hw->mac.addr, vid, set);
992 if (err)
993 goto err_out;
994
995
996 interface->vid = vid + (set ? VLAN_N_VID : 0);
997
998
999 __dev_uc_unsync(netdev, fm10k_uc_vlan_unsync);
1000 __dev_mc_unsync(netdev, fm10k_mc_vlan_unsync);
1001
1002err_out:
1003 fm10k_mbx_unlock(interface);
1004
1005 return err;
1006}
1007
1008static int fm10k_vlan_rx_add_vid(struct net_device *netdev,
1009 __always_unused __be16 proto, u16 vid)
1010{
1011
1012 return fm10k_update_vid(netdev, vid, true);
1013}
1014
1015static int fm10k_vlan_rx_kill_vid(struct net_device *netdev,
1016 __always_unused __be16 proto, u16 vid)
1017{
1018
1019 return fm10k_update_vid(netdev, vid, false);
1020}
1021
1022static u16 fm10k_find_next_vlan(struct fm10k_intfc *interface, u16 vid)
1023{
1024 struct fm10k_hw *hw = &interface->hw;
1025 u16 default_vid = hw->mac.default_vid;
1026 u16 vid_limit = vid < default_vid ? default_vid : VLAN_N_VID;
1027
1028 vid = find_next_bit(interface->active_vlans, vid_limit, ++vid);
1029
1030 return vid;
1031}
1032
1033static void fm10k_clear_unused_vlans(struct fm10k_intfc *interface)
1034{
1035 u32 vid, prev_vid;
1036
1037
1038 for (vid = 0, prev_vid = 0;
1039 prev_vid < VLAN_N_VID;
1040 prev_vid = vid + 1, vid = fm10k_find_next_vlan(interface, vid)) {
1041 if (prev_vid == vid)
1042 continue;
1043
1044
1045 prev_vid += (vid - prev_vid - 1) << FM10K_VLAN_LENGTH_SHIFT;
1046 fm10k_queue_vlan_request(interface, prev_vid, 0, false);
1047 }
1048}
1049
1050static int __fm10k_uc_sync(struct net_device *dev,
1051 const unsigned char *addr, bool sync)
1052{
1053 struct fm10k_intfc *interface = netdev_priv(dev);
1054 u16 vid, glort = interface->glort;
1055 s32 err;
1056
1057 if (!is_valid_ether_addr(addr))
1058 return -EADDRNOTAVAIL;
1059
1060 for (vid = fm10k_find_next_vlan(interface, 0);
1061 vid < VLAN_N_VID;
1062 vid = fm10k_find_next_vlan(interface, vid)) {
1063 err = fm10k_queue_mac_request(interface, glort,
1064 addr, vid, sync);
1065 if (err)
1066 return err;
1067 }
1068
1069 return 0;
1070}
1071
1072static int fm10k_uc_sync(struct net_device *dev,
1073 const unsigned char *addr)
1074{
1075 return __fm10k_uc_sync(dev, addr, true);
1076}
1077
1078static int fm10k_uc_unsync(struct net_device *dev,
1079 const unsigned char *addr)
1080{
1081 return __fm10k_uc_sync(dev, addr, false);
1082}
1083
1084static int fm10k_set_mac(struct net_device *dev, void *p)
1085{
1086 struct fm10k_intfc *interface = netdev_priv(dev);
1087 struct fm10k_hw *hw = &interface->hw;
1088 struct sockaddr *addr = p;
1089 s32 err = 0;
1090
1091 if (!is_valid_ether_addr(addr->sa_data))
1092 return -EADDRNOTAVAIL;
1093
1094 if (dev->flags & IFF_UP) {
1095
1096 fm10k_mbx_lock(interface);
1097
1098 err = fm10k_uc_sync(dev, addr->sa_data);
1099 if (!err)
1100 fm10k_uc_unsync(dev, hw->mac.addr);
1101
1102 fm10k_mbx_unlock(interface);
1103 }
1104
1105 if (!err) {
1106 ether_addr_copy(dev->dev_addr, addr->sa_data);
1107 ether_addr_copy(hw->mac.addr, addr->sa_data);
1108 dev->addr_assign_type &= ~NET_ADDR_RANDOM;
1109 }
1110
1111
1112 return err ? -EAGAIN : 0;
1113}
1114
1115static int __fm10k_mc_sync(struct net_device *dev,
1116 const unsigned char *addr, bool sync)
1117{
1118 struct fm10k_intfc *interface = netdev_priv(dev);
1119 u16 vid, glort = interface->glort;
1120 s32 err;
1121
1122 if (!is_multicast_ether_addr(addr))
1123 return -EADDRNOTAVAIL;
1124
1125 for (vid = fm10k_find_next_vlan(interface, 0);
1126 vid < VLAN_N_VID;
1127 vid = fm10k_find_next_vlan(interface, vid)) {
1128 err = fm10k_queue_mac_request(interface, glort,
1129 addr, vid, sync);
1130 if (err)
1131 return err;
1132 }
1133
1134 return 0;
1135}
1136
1137static int fm10k_mc_sync(struct net_device *dev,
1138 const unsigned char *addr)
1139{
1140 return __fm10k_mc_sync(dev, addr, true);
1141}
1142
1143static int fm10k_mc_unsync(struct net_device *dev,
1144 const unsigned char *addr)
1145{
1146 return __fm10k_mc_sync(dev, addr, false);
1147}
1148
1149static void fm10k_set_rx_mode(struct net_device *dev)
1150{
1151 struct fm10k_intfc *interface = netdev_priv(dev);
1152 struct fm10k_hw *hw = &interface->hw;
1153 int xcast_mode;
1154
1155
1156 if (!(dev->flags & IFF_UP))
1157 return;
1158
1159
1160 xcast_mode = (dev->flags & IFF_PROMISC) ? FM10K_XCAST_MODE_PROMISC :
1161 (dev->flags & IFF_ALLMULTI) ? FM10K_XCAST_MODE_ALLMULTI :
1162 (dev->flags & (IFF_BROADCAST | IFF_MULTICAST)) ?
1163 FM10K_XCAST_MODE_MULTI : FM10K_XCAST_MODE_NONE;
1164
1165 fm10k_mbx_lock(interface);
1166
1167
1168 if (interface->xcast_mode != xcast_mode) {
1169
1170 if (xcast_mode == FM10K_XCAST_MODE_PROMISC)
1171 fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL,
1172 0, true);
1173
1174
1175 if (interface->xcast_mode == FM10K_XCAST_MODE_PROMISC)
1176 fm10k_clear_unused_vlans(interface);
1177
1178
1179 if (fm10k_host_mbx_ready(interface))
1180 hw->mac.ops.update_xcast_mode(hw, interface->glort,
1181 xcast_mode);
1182
1183
1184 interface->xcast_mode = xcast_mode;
1185 }
1186
1187
1188 __dev_uc_sync(dev, fm10k_uc_sync, fm10k_uc_unsync);
1189 __dev_mc_sync(dev, fm10k_mc_sync, fm10k_mc_unsync);
1190
1191 fm10k_mbx_unlock(interface);
1192}
1193
1194void fm10k_restore_rx_state(struct fm10k_intfc *interface)
1195{
1196 struct fm10k_l2_accel *l2_accel = interface->l2_accel;
1197 struct net_device *netdev = interface->netdev;
1198 struct fm10k_hw *hw = &interface->hw;
1199 int xcast_mode, i;
1200 u16 vid, glort;
1201
1202
1203 glort = interface->glort;
1204
1205
1206 if (netdev->flags & IFF_PROMISC)
1207 xcast_mode = FM10K_XCAST_MODE_PROMISC;
1208 else if (netdev->flags & IFF_ALLMULTI)
1209 xcast_mode = FM10K_XCAST_MODE_ALLMULTI;
1210 else if (netdev->flags & (IFF_BROADCAST | IFF_MULTICAST))
1211 xcast_mode = FM10K_XCAST_MODE_MULTI;
1212 else
1213 xcast_mode = FM10K_XCAST_MODE_NONE;
1214
1215 fm10k_mbx_lock(interface);
1216
1217
1218 if (fm10k_host_mbx_ready(interface))
1219 hw->mac.ops.update_lport_state(hw, glort,
1220 interface->glort_count, true);
1221
1222
1223 fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL, 0,
1224 xcast_mode == FM10K_XCAST_MODE_PROMISC);
1225
1226
1227 for (vid = fm10k_find_next_vlan(interface, 0);
1228 vid < VLAN_N_VID;
1229 vid = fm10k_find_next_vlan(interface, vid)) {
1230 fm10k_queue_vlan_request(interface, vid, 0, true);
1231
1232 fm10k_queue_mac_request(interface, glort,
1233 hw->mac.addr, vid, true);
1234 }
1235
1236
1237
1238
1239 if (fm10k_host_mbx_ready(interface))
1240 hw->mac.ops.update_xcast_mode(hw, glort, xcast_mode);
1241
1242
1243 __dev_uc_sync(netdev, fm10k_uc_sync, fm10k_uc_unsync);
1244 __dev_mc_sync(netdev, fm10k_mc_sync, fm10k_mc_unsync);
1245
1246
1247 if (l2_accel) {
1248 for (i = 0; i < l2_accel->size; i++) {
1249 struct net_device *sdev = l2_accel->macvlan[i];
1250
1251 if (!sdev)
1252 continue;
1253
1254 glort = l2_accel->dglort + 1 + i;
1255
1256 hw->mac.ops.update_xcast_mode(hw, glort,
1257 FM10K_XCAST_MODE_MULTI);
1258 fm10k_queue_mac_request(interface, glort,
1259 sdev->dev_addr,
1260 hw->mac.default_vid, true);
1261 }
1262 }
1263
1264 fm10k_mbx_unlock(interface);
1265
1266
1267 interface->xcast_mode = xcast_mode;
1268
1269
1270 fm10k_restore_udp_port_info(interface);
1271}
1272
1273void fm10k_reset_rx_state(struct fm10k_intfc *interface)
1274{
1275 struct net_device *netdev = interface->netdev;
1276 struct fm10k_hw *hw = &interface->hw;
1277
1278
1279 while (test_bit(__FM10K_MACVLAN_SCHED, interface->state))
1280 usleep_range(1000, 2000);
1281
1282
1283 fm10k_clear_macvlan_queue(interface, interface->glort, true);
1284
1285 fm10k_mbx_lock(interface);
1286
1287
1288
1289
1290 if (fm10k_host_mbx_ready(interface))
1291 hw->mac.ops.update_lport_state(hw, interface->glort,
1292 interface->glort_count, false);
1293
1294 fm10k_mbx_unlock(interface);
1295
1296
1297 interface->xcast_mode = FM10K_XCAST_MODE_NONE;
1298
1299
1300 __dev_uc_unsync(netdev, NULL);
1301 __dev_mc_unsync(netdev, NULL);
1302}
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312static void fm10k_get_stats64(struct net_device *netdev,
1313 struct rtnl_link_stats64 *stats)
1314{
1315 struct fm10k_intfc *interface = netdev_priv(netdev);
1316 struct fm10k_ring *ring;
1317 unsigned int start, i;
1318 u64 bytes, packets;
1319
1320 rcu_read_lock();
1321
1322 for (i = 0; i < interface->num_rx_queues; i++) {
1323 ring = READ_ONCE(interface->rx_ring[i]);
1324
1325 if (!ring)
1326 continue;
1327
1328 do {
1329 start = u64_stats_fetch_begin_irq(&ring->syncp);
1330 packets = ring->stats.packets;
1331 bytes = ring->stats.bytes;
1332 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1333
1334 stats->rx_packets += packets;
1335 stats->rx_bytes += bytes;
1336 }
1337
1338 for (i = 0; i < interface->num_tx_queues; i++) {
1339 ring = READ_ONCE(interface->tx_ring[i]);
1340
1341 if (!ring)
1342 continue;
1343
1344 do {
1345 start = u64_stats_fetch_begin_irq(&ring->syncp);
1346 packets = ring->stats.packets;
1347 bytes = ring->stats.bytes;
1348 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1349
1350 stats->tx_packets += packets;
1351 stats->tx_bytes += bytes;
1352 }
1353
1354 rcu_read_unlock();
1355
1356
1357 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1358}
1359
1360int fm10k_setup_tc(struct net_device *dev, u8 tc)
1361{
1362 struct fm10k_intfc *interface = netdev_priv(dev);
1363 int err;
1364
1365
1366 if (tc && (interface->hw.mac.type != fm10k_mac_pf))
1367 return -EINVAL;
1368
1369
1370 if (tc > 8)
1371 return -EINVAL;
1372
1373
1374
1375
1376
1377 if (netif_running(dev))
1378 fm10k_close(dev);
1379
1380 fm10k_mbx_free_irq(interface);
1381
1382 fm10k_clear_queueing_scheme(interface);
1383
1384
1385 netdev_reset_tc(dev);
1386 netdev_set_num_tc(dev, tc);
1387
1388 err = fm10k_init_queueing_scheme(interface);
1389 if (err)
1390 goto err_queueing_scheme;
1391
1392 err = fm10k_mbx_request_irq(interface);
1393 if (err)
1394 goto err_mbx_irq;
1395
1396 err = netif_running(dev) ? fm10k_open(dev) : 0;
1397 if (err)
1398 goto err_open;
1399
1400
1401 set_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags);
1402
1403 return 0;
1404err_open:
1405 fm10k_mbx_free_irq(interface);
1406err_mbx_irq:
1407 fm10k_clear_queueing_scheme(interface);
1408err_queueing_scheme:
1409 netif_device_detach(dev);
1410
1411 return err;
1412}
1413
1414static int __fm10k_setup_tc(struct net_device *dev, enum tc_setup_type type,
1415 void *type_data)
1416{
1417 struct tc_mqprio_qopt *mqprio = type_data;
1418
1419 if (type != TC_SETUP_QDISC_MQPRIO)
1420 return -EOPNOTSUPP;
1421
1422 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1423
1424 return fm10k_setup_tc(dev, mqprio->num_tc);
1425}
1426
1427static void fm10k_assign_l2_accel(struct fm10k_intfc *interface,
1428 struct fm10k_l2_accel *l2_accel)
1429{
1430 struct fm10k_ring *ring;
1431 int i;
1432
1433 for (i = 0; i < interface->num_rx_queues; i++) {
1434 ring = interface->rx_ring[i];
1435 rcu_assign_pointer(ring->l2_accel, l2_accel);
1436 }
1437
1438 interface->l2_accel = l2_accel;
1439}
1440
1441static void *fm10k_dfwd_add_station(struct net_device *dev,
1442 struct net_device *sdev)
1443{
1444 struct fm10k_intfc *interface = netdev_priv(dev);
1445 struct fm10k_l2_accel *l2_accel = interface->l2_accel;
1446 struct fm10k_l2_accel *old_l2_accel = NULL;
1447 struct fm10k_dglort_cfg dglort = { 0 };
1448 struct fm10k_hw *hw = &interface->hw;
1449 int size = 0, i;
1450 u16 glort;
1451
1452
1453 if (!l2_accel) {
1454
1455 if (interface->glort_count < 7)
1456 return ERR_PTR(-EBUSY);
1457
1458 size = offsetof(struct fm10k_l2_accel, macvlan[7]);
1459 l2_accel = kzalloc(size, GFP_KERNEL);
1460 if (!l2_accel)
1461 return ERR_PTR(-ENOMEM);
1462
1463 l2_accel->size = 7;
1464 l2_accel->dglort = interface->glort;
1465
1466
1467 fm10k_assign_l2_accel(interface, l2_accel);
1468
1469 } else if ((l2_accel->count == FM10K_MAX_STATIONS) ||
1470 (l2_accel->count == (interface->glort_count - 1))) {
1471 return ERR_PTR(-EBUSY);
1472
1473 } else if (l2_accel->count == l2_accel->size) {
1474 old_l2_accel = l2_accel;
1475 size = offsetof(struct fm10k_l2_accel,
1476 macvlan[(l2_accel->size * 2) + 1]);
1477 l2_accel = kzalloc(size, GFP_KERNEL);
1478 if (!l2_accel)
1479 return ERR_PTR(-ENOMEM);
1480
1481 memcpy(l2_accel, old_l2_accel,
1482 offsetof(struct fm10k_l2_accel,
1483 macvlan[old_l2_accel->size]));
1484
1485 l2_accel->size = (old_l2_accel->size * 2) + 1;
1486
1487
1488 fm10k_assign_l2_accel(interface, l2_accel);
1489 kfree_rcu(old_l2_accel, rcu);
1490 }
1491
1492
1493 for (i = 0; i < l2_accel->size; i++) {
1494 if (!l2_accel->macvlan[i])
1495 break;
1496 }
1497
1498
1499 l2_accel->macvlan[i] = sdev;
1500 l2_accel->count++;
1501
1502
1503 dglort.idx = fm10k_dglort_pf_rss;
1504 dglort.inner_rss = 1;
1505 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1506 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1507 dglort.glort = interface->glort;
1508 dglort.shared_l = fls(l2_accel->size);
1509 hw->mac.ops.configure_dglort_map(hw, &dglort);
1510
1511
1512 fm10k_mbx_lock(interface);
1513
1514 glort = l2_accel->dglort + 1 + i;
1515
1516 if (fm10k_host_mbx_ready(interface)) {
1517 hw->mac.ops.update_xcast_mode(hw, glort,
1518 FM10K_XCAST_MODE_MULTI);
1519 fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
1520 hw->mac.default_vid, true);
1521 }
1522
1523 fm10k_mbx_unlock(interface);
1524
1525 return sdev;
1526}
1527
1528static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
1529{
1530 struct fm10k_intfc *interface = netdev_priv(dev);
1531 struct fm10k_l2_accel *l2_accel = READ_ONCE(interface->l2_accel);
1532 struct fm10k_dglort_cfg dglort = { 0 };
1533 struct fm10k_hw *hw = &interface->hw;
1534 struct net_device *sdev = priv;
1535 int i;
1536 u16 glort;
1537
1538 if (!l2_accel)
1539 return;
1540
1541
1542 for (i = 0; i < l2_accel->size; i++) {
1543 if (l2_accel->macvlan[i] == sdev)
1544 break;
1545 }
1546
1547
1548 if (i == l2_accel->size)
1549 return;
1550
1551
1552 fm10k_mbx_lock(interface);
1553
1554 glort = l2_accel->dglort + 1 + i;
1555
1556 if (fm10k_host_mbx_ready(interface)) {
1557 hw->mac.ops.update_xcast_mode(hw, glort,
1558 FM10K_XCAST_MODE_NONE);
1559 fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
1560 hw->mac.default_vid, false);
1561 }
1562
1563 fm10k_mbx_unlock(interface);
1564
1565
1566 l2_accel->macvlan[i] = NULL;
1567 l2_accel->count--;
1568
1569
1570 dglort.idx = fm10k_dglort_pf_rss;
1571 dglort.inner_rss = 1;
1572 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1573 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1574 dglort.glort = interface->glort;
1575 dglort.shared_l = fls(l2_accel->size);
1576 hw->mac.ops.configure_dglort_map(hw, &dglort);
1577
1578
1579 if (l2_accel->count == 0) {
1580 fm10k_assign_l2_accel(interface, NULL);
1581 kfree_rcu(l2_accel, rcu);
1582 }
1583}
1584
1585static netdev_features_t fm10k_features_check(struct sk_buff *skb,
1586 struct net_device *dev,
1587 netdev_features_t features)
1588{
1589 if (!skb->encapsulation || fm10k_tx_encap_offload(skb))
1590 return features;
1591
1592 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1593}
1594
1595static const struct net_device_ops fm10k_netdev_ops = {
1596 .ndo_open = fm10k_open,
1597 .ndo_stop = fm10k_close,
1598 .ndo_validate_addr = eth_validate_addr,
1599 .ndo_start_xmit = fm10k_xmit_frame,
1600 .ndo_set_mac_address = fm10k_set_mac,
1601 .ndo_tx_timeout = fm10k_tx_timeout,
1602 .ndo_vlan_rx_add_vid = fm10k_vlan_rx_add_vid,
1603 .ndo_vlan_rx_kill_vid = fm10k_vlan_rx_kill_vid,
1604 .ndo_set_rx_mode = fm10k_set_rx_mode,
1605 .ndo_get_stats64 = fm10k_get_stats64,
1606 .ndo_setup_tc = __fm10k_setup_tc,
1607 .ndo_set_vf_mac = fm10k_ndo_set_vf_mac,
1608 .ndo_set_vf_vlan = fm10k_ndo_set_vf_vlan,
1609 .ndo_set_vf_rate = fm10k_ndo_set_vf_bw,
1610 .ndo_get_vf_config = fm10k_ndo_get_vf_config,
1611 .ndo_udp_tunnel_add = fm10k_udp_tunnel_add,
1612 .ndo_udp_tunnel_del = fm10k_udp_tunnel_del,
1613 .ndo_dfwd_add_station = fm10k_dfwd_add_station,
1614 .ndo_dfwd_del_station = fm10k_dfwd_del_station,
1615#ifdef CONFIG_NET_POLL_CONTROLLER
1616 .ndo_poll_controller = fm10k_netpoll,
1617#endif
1618 .ndo_features_check = fm10k_features_check,
1619};
1620
1621#define DEFAULT_DEBUG_LEVEL_SHIFT 3
1622
1623struct net_device *fm10k_alloc_netdev(const struct fm10k_info *info)
1624{
1625 netdev_features_t hw_features;
1626 struct fm10k_intfc *interface;
1627 struct net_device *dev;
1628
1629 dev = alloc_etherdev_mq(sizeof(struct fm10k_intfc), MAX_QUEUES);
1630 if (!dev)
1631 return NULL;
1632
1633
1634 dev->netdev_ops = &fm10k_netdev_ops;
1635 fm10k_set_ethtool_ops(dev);
1636
1637
1638 interface = netdev_priv(dev);
1639 interface->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
1640
1641
1642 dev->features |= NETIF_F_IP_CSUM |
1643 NETIF_F_IPV6_CSUM |
1644 NETIF_F_SG |
1645 NETIF_F_TSO |
1646 NETIF_F_TSO6 |
1647 NETIF_F_TSO_ECN |
1648 NETIF_F_RXHASH |
1649 NETIF_F_RXCSUM;
1650
1651
1652 if (info->mac == fm10k_mac_pf) {
1653 dev->hw_enc_features = NETIF_F_IP_CSUM |
1654 NETIF_F_TSO |
1655 NETIF_F_TSO6 |
1656 NETIF_F_TSO_ECN |
1657 NETIF_F_GSO_UDP_TUNNEL |
1658 NETIF_F_IPV6_CSUM |
1659 NETIF_F_SG;
1660
1661 dev->features |= NETIF_F_GSO_UDP_TUNNEL;
1662 }
1663
1664
1665 hw_features = dev->features;
1666
1667
1668 hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
1669
1670
1671 dev->vlan_features |= dev->features;
1672
1673
1674
1675
1676
1677 dev->features |= NETIF_F_HW_VLAN_CTAG_TX |
1678 NETIF_F_HW_VLAN_CTAG_RX |
1679 NETIF_F_HW_VLAN_CTAG_FILTER;
1680
1681 dev->priv_flags |= IFF_UNICAST_FLT;
1682
1683 dev->hw_features |= hw_features;
1684
1685
1686 dev->min_mtu = ETH_MIN_MTU;
1687 dev->max_mtu = FM10K_MAX_JUMBO_FRAME_SIZE;
1688
1689 return dev;
1690}
1691