1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#include <linux/skbuff.h>
37#include <linux/netdevice.h>
38#include <linux/etherdevice.h>
39#include <linux/if_vlan.h>
40#include <linux/ip.h>
41#include <net/ipv6.h>
42#include <net/tcp.h>
43#include <linux/dma-mapping.h>
44#include <linux/prefetch.h>
45
46#include "t4vf_common.h"
47#include "t4vf_defs.h"
48
49#include "../cxgb4/t4_regs.h"
50#include "../cxgb4/t4fw_api.h"
51#include "../cxgb4/t4_msg.h"
52
53
54
55
56static u32 FL_PG_ORDER;
57static u32 STAT_LEN;
58static u32 PKTSHIFT;
59static u32 FL_ALIGN;
60
61
62
63
64enum {
65
66
67
68
69
70
71
72 EQ_UNIT = SGE_EQ_IDXSIZE,
73 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
74 TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
75
76
77
78
79
80
81
82 MAX_TX_RECLAIM = 16,
83
84
85
86
87
88 MAX_RX_REFILL = 16,
89
90
91
92
93
94
95 RX_QCHECK_PERIOD = (HZ / 2),
96
97
98
99
100
101 TX_QCHECK_PERIOD = (HZ / 2),
102 MAX_TIMER_TX_RECLAIM = 100,
103
104
105
106
107
108 FL_STARVE_THRES = 4,
109
110
111
112
113
114
115
116
117
118 ETHTXQ_MAX_FRAGS = MAX_SKB_FRAGS + 1,
119 ETHTXQ_MAX_SGL_LEN = ((3 * (ETHTXQ_MAX_FRAGS-1))/2 +
120 ((ETHTXQ_MAX_FRAGS-1) & 1) +
121 2),
122 ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
123 sizeof(struct cpl_tx_pkt_lso_core) +
124 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
125 ETHTXQ_MAX_FLITS = ETHTXQ_MAX_SGL_LEN + ETHTXQ_MAX_HDR,
126
127 ETHTXQ_STOP_THRES = 1 + DIV_ROUND_UP(ETHTXQ_MAX_FLITS, TXD_PER_EQ_UNIT),
128
129
130
131
132
133
134
135 MAX_IMM_TX_PKT_LEN = FW_WR_IMMDLEN_MASK,
136
137
138
139
140 MAX_CTRL_WR_LEN = 256,
141
142
143
144
145
146 MAX_IMM_TX_LEN = (MAX_IMM_TX_PKT_LEN > MAX_CTRL_WR_LEN
147 ? MAX_IMM_TX_PKT_LEN
148 : MAX_CTRL_WR_LEN),
149
150
151
152
153
154
155
156 RX_COPY_THRES = 256,
157 RX_PULL_LEN = 128,
158
159
160
161
162
163
164 RX_SKB_LEN = 512,
165};
166
167
168
169
170struct tx_sw_desc {
171 struct sk_buff *skb;
172 struct ulptx_sgl *sgl;
173};
174
175
176
177
178
179
180
181struct rx_sw_desc {
182 struct page *page;
183 dma_addr_t dma_addr;
184
185};
186
187
188
189
190
191
192
193
194
195
196enum {
197 RX_LARGE_BUF = 1 << 0,
198 RX_UNMAPPED_BUF = 1 << 1,
199};
200
201
202
203
204
205
206
207
208static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *sdesc)
209{
210 return sdesc->dma_addr & ~(dma_addr_t)(RX_LARGE_BUF | RX_UNMAPPED_BUF);
211}
212
213
214
215
216
217
218
219
220static inline bool is_buf_mapped(const struct rx_sw_desc *sdesc)
221{
222 return !(sdesc->dma_addr & RX_UNMAPPED_BUF);
223}
224
225
226
227
228
229
230
231static inline int need_skb_unmap(void)
232{
233#ifdef CONFIG_NEED_DMA_MAP_STATE
234 return 1;
235#else
236 return 0;
237#endif
238}
239
240
241
242
243
244
245
246static inline unsigned int txq_avail(const struct sge_txq *tq)
247{
248 return tq->size - 1 - tq->in_use;
249}
250
251
252
253
254
255
256
257
258
259
260static inline unsigned int fl_cap(const struct sge_fl *fl)
261{
262 return fl->size - FL_PER_EQ_UNIT;
263}
264
265
266
267
268
269
270
271
272
273static inline bool fl_starving(const struct sge_fl *fl)
274{
275 return fl->avail - fl->pend_cred <= FL_STARVE_THRES;
276}
277
278
279
280
281
282
283
284
285
286static int map_skb(struct device *dev, const struct sk_buff *skb,
287 dma_addr_t *addr)
288{
289 const skb_frag_t *fp, *end;
290 const struct skb_shared_info *si;
291
292 *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
293 if (dma_mapping_error(dev, *addr))
294 goto out_err;
295
296 si = skb_shinfo(skb);
297 end = &si->frags[si->nr_frags];
298 for (fp = si->frags; fp < end; fp++) {
299 *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
300 DMA_TO_DEVICE);
301 if (dma_mapping_error(dev, *addr))
302 goto unwind;
303 }
304 return 0;
305
306unwind:
307 while (fp-- > si->frags)
308 dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
309 dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
310
311out_err:
312 return -ENOMEM;
313}
314
315static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
316 const struct ulptx_sgl *sgl, const struct sge_txq *tq)
317{
318 const struct ulptx_sge_pair *p;
319 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
320
321 if (likely(skb_headlen(skb)))
322 dma_unmap_single(dev, be64_to_cpu(sgl->addr0),
323 be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
324 else {
325 dma_unmap_page(dev, be64_to_cpu(sgl->addr0),
326 be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
327 nfrags--;
328 }
329
330
331
332
333
334 for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
335 if (likely((u8 *)(p + 1) <= (u8 *)tq->stat)) {
336unmap:
337 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
338 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
339 dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
340 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
341 p++;
342 } else if ((u8 *)p == (u8 *)tq->stat) {
343 p = (const struct ulptx_sge_pair *)tq->desc;
344 goto unmap;
345 } else if ((u8 *)p + 8 == (u8 *)tq->stat) {
346 const __be64 *addr = (const __be64 *)tq->desc;
347
348 dma_unmap_page(dev, be64_to_cpu(addr[0]),
349 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
350 dma_unmap_page(dev, be64_to_cpu(addr[1]),
351 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
352 p = (const struct ulptx_sge_pair *)&addr[2];
353 } else {
354 const __be64 *addr = (const __be64 *)tq->desc;
355
356 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
357 be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
358 dma_unmap_page(dev, be64_to_cpu(addr[0]),
359 be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
360 p = (const struct ulptx_sge_pair *)&addr[1];
361 }
362 }
363 if (nfrags) {
364 __be64 addr;
365
366 if ((u8 *)p == (u8 *)tq->stat)
367 p = (const struct ulptx_sge_pair *)tq->desc;
368 addr = ((u8 *)p + 16 <= (u8 *)tq->stat
369 ? p->addr[0]
370 : *(const __be64 *)tq->desc);
371 dma_unmap_page(dev, be64_to_cpu(addr), be32_to_cpu(p->len[0]),
372 DMA_TO_DEVICE);
373 }
374}
375
376
377
378
379
380
381
382
383
384
385
386static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq,
387 unsigned int n, bool unmap)
388{
389 struct tx_sw_desc *sdesc;
390 unsigned int cidx = tq->cidx;
391 struct device *dev = adapter->pdev_dev;
392
393 const int need_unmap = need_skb_unmap() && unmap;
394
395 sdesc = &tq->sdesc[cidx];
396 while (n--) {
397
398
399
400
401 if (sdesc->skb) {
402 if (need_unmap)
403 unmap_sgl(dev, sdesc->skb, sdesc->sgl, tq);
404 kfree_skb(sdesc->skb);
405 sdesc->skb = NULL;
406 }
407
408 sdesc++;
409 if (++cidx == tq->size) {
410 cidx = 0;
411 sdesc = tq->sdesc;
412 }
413 }
414 tq->cidx = cidx;
415}
416
417
418
419
420static inline int reclaimable(const struct sge_txq *tq)
421{
422 int hw_cidx = be16_to_cpu(tq->stat->cidx);
423 int reclaimable = hw_cidx - tq->cidx;
424 if (reclaimable < 0)
425 reclaimable += tq->size;
426 return reclaimable;
427}
428
429
430
431
432
433
434
435
436
437
438
439static inline void reclaim_completed_tx(struct adapter *adapter,
440 struct sge_txq *tq,
441 bool unmap)
442{
443 int avail = reclaimable(tq);
444
445 if (avail) {
446
447
448
449
450 if (avail > MAX_TX_RECLAIM)
451 avail = MAX_TX_RECLAIM;
452
453 free_tx_desc(adapter, tq, avail, unmap);
454 tq->in_use -= avail;
455 }
456}
457
458
459
460
461
462static inline int get_buf_size(const struct rx_sw_desc *sdesc)
463{
464 return FL_PG_ORDER > 0 && (sdesc->dma_addr & RX_LARGE_BUF)
465 ? (PAGE_SIZE << FL_PG_ORDER)
466 : PAGE_SIZE;
467}
468
469
470
471
472
473
474
475
476
477
478
479static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n)
480{
481 while (n--) {
482 struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
483
484 if (is_buf_mapped(sdesc))
485 dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
486 get_buf_size(sdesc), PCI_DMA_FROMDEVICE);
487 put_page(sdesc->page);
488 sdesc->page = NULL;
489 if (++fl->cidx == fl->size)
490 fl->cidx = 0;
491 fl->avail--;
492 }
493}
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl)
509{
510 struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
511
512 if (is_buf_mapped(sdesc))
513 dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
514 get_buf_size(sdesc), PCI_DMA_FROMDEVICE);
515 sdesc->page = NULL;
516 if (++fl->cidx == fl->size)
517 fl->cidx = 0;
518 fl->avail--;
519}
520
521
522
523
524
525
526
527
528
529static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl)
530{
531 u32 val;
532
533
534
535
536
537
538 if (fl->pend_cred >= FL_PER_EQ_UNIT) {
539 val = PIDX(fl->pend_cred / FL_PER_EQ_UNIT);
540 if (!is_t4(adapter->chip))
541 val |= DBTYPE(1);
542 wmb();
543 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
544 DBPRIO(1) |
545 QID(fl->cntxt_id) | val);
546 fl->pend_cred %= FL_PER_EQ_UNIT;
547 }
548}
549
550
551
552
553
554
555
556static inline void set_rx_sw_desc(struct rx_sw_desc *sdesc, struct page *page,
557 dma_addr_t dma_addr)
558{
559 sdesc->page = page;
560 sdesc->dma_addr = dma_addr;
561}
562
563
564
565
566#define POISON_BUF_VAL -1
567
568static inline void poison_buf(struct page *page, size_t sz)
569{
570#if POISON_BUF_VAL >= 0
571 memset(page_address(page), POISON_BUF_VAL, sz);
572#endif
573}
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,
590 int n, gfp_t gfp)
591{
592 struct page *page;
593 dma_addr_t dma_addr;
594 unsigned int cred = fl->avail;
595 __be64 *d = &fl->desc[fl->pidx];
596 struct rx_sw_desc *sdesc = &fl->sdesc[fl->pidx];
597
598
599
600
601
602
603 BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT);
604
605
606
607
608
609
610
611 if (FL_PG_ORDER == 0)
612 goto alloc_small_pages;
613
614 while (n) {
615 page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN,
616 FL_PG_ORDER);
617 if (unlikely(!page)) {
618
619
620
621
622
623 fl->large_alloc_failed++;
624 break;
625 }
626 poison_buf(page, PAGE_SIZE << FL_PG_ORDER);
627
628 dma_addr = dma_map_page(adapter->pdev_dev, page, 0,
629 PAGE_SIZE << FL_PG_ORDER,
630 PCI_DMA_FROMDEVICE);
631 if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
632
633
634
635
636
637
638
639
640 __free_pages(page, FL_PG_ORDER);
641 goto out;
642 }
643 dma_addr |= RX_LARGE_BUF;
644 *d++ = cpu_to_be64(dma_addr);
645
646 set_rx_sw_desc(sdesc, page, dma_addr);
647 sdesc++;
648
649 fl->avail++;
650 if (++fl->pidx == fl->size) {
651 fl->pidx = 0;
652 sdesc = fl->sdesc;
653 d = fl->desc;
654 }
655 n--;
656 }
657
658alloc_small_pages:
659 while (n--) {
660 page = __skb_alloc_page(gfp | __GFP_NOWARN, NULL);
661 if (unlikely(!page)) {
662 fl->alloc_failed++;
663 break;
664 }
665 poison_buf(page, PAGE_SIZE);
666
667 dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE,
668 PCI_DMA_FROMDEVICE);
669 if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
670 put_page(page);
671 break;
672 }
673 *d++ = cpu_to_be64(dma_addr);
674
675 set_rx_sw_desc(sdesc, page, dma_addr);
676 sdesc++;
677
678 fl->avail++;
679 if (++fl->pidx == fl->size) {
680 fl->pidx = 0;
681 sdesc = fl->sdesc;
682 d = fl->desc;
683 }
684 }
685
686out:
687
688
689
690
691
692 cred = fl->avail - cred;
693 fl->pend_cred += cred;
694 ring_fl_db(adapter, fl);
695
696 if (unlikely(fl_starving(fl))) {
697 smp_wmb();
698 set_bit(fl->cntxt_id, adapter->sge.starving_fl);
699 }
700
701 return cred;
702}
703
704
705
706
707
708static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl)
709{
710 refill_fl(adapter, fl,
711 min((unsigned int)MAX_RX_REFILL, fl_cap(fl) - fl->avail),
712 GFP_ATOMIC);
713}
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734static void *alloc_ring(struct device *dev, size_t nelem, size_t hwsize,
735 size_t swsize, dma_addr_t *busaddrp, void *swringp,
736 size_t stat_size)
737{
738
739
740
741 size_t hwlen = nelem * hwsize + stat_size;
742 void *hwring = dma_alloc_coherent(dev, hwlen, busaddrp, GFP_KERNEL);
743
744 if (!hwring)
745 return NULL;
746
747
748
749
750
751 BUG_ON((swsize != 0) != (swringp != NULL));
752 if (swsize) {
753 void *swring = kcalloc(nelem, swsize, GFP_KERNEL);
754
755 if (!swring) {
756 dma_free_coherent(dev, hwlen, hwring, *busaddrp);
757 return NULL;
758 }
759 *(void **)swringp = swring;
760 }
761
762
763
764
765
766 memset(hwring, 0, hwlen);
767 return hwring;
768}
769
770
771
772
773
774
775
776
777static inline unsigned int sgl_len(unsigned int n)
778{
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796 n--;
797 return (3 * n) / 2 + (n & 1) + 2;
798}
799
800
801
802
803
804
805
806
807static inline unsigned int flits_to_desc(unsigned int flits)
808{
809 BUG_ON(flits > SGE_MAX_WR_LEN / sizeof(__be64));
810 return DIV_ROUND_UP(flits, TXD_PER_EQ_UNIT);
811}
812
813
814
815
816
817
818
819
820static inline int is_eth_imm(const struct sk_buff *skb)
821{
822
823
824
825
826
827
828
829 return false;
830}
831
832
833
834
835
836
837
838
839static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
840{
841 unsigned int flits;
842
843
844
845
846
847
848 if (is_eth_imm(skb))
849 return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
850 sizeof(__be64));
851
852
853
854
855
856
857
858
859
860
861 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
862 if (skb_shinfo(skb)->gso_size)
863 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
864 sizeof(struct cpl_tx_pkt_lso_core) +
865 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
866 else
867 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
868 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
869 return flits;
870}
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889static void write_sgl(const struct sk_buff *skb, struct sge_txq *tq,
890 struct ulptx_sgl *sgl, u64 *end, unsigned int start,
891 const dma_addr_t *addr)
892{
893 unsigned int i, len;
894 struct ulptx_sge_pair *to;
895 const struct skb_shared_info *si = skb_shinfo(skb);
896 unsigned int nfrags = si->nr_frags;
897 struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
898
899 len = skb_headlen(skb) - start;
900 if (likely(len)) {
901 sgl->len0 = htonl(len);
902 sgl->addr0 = cpu_to_be64(addr[0] + start);
903 nfrags++;
904 } else {
905 sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
906 sgl->addr0 = cpu_to_be64(addr[1]);
907 }
908
909 sgl->cmd_nsge = htonl(ULPTX_CMD(ULP_TX_SC_DSGL) |
910 ULPTX_NSGE(nfrags));
911 if (likely(--nfrags == 0))
912 return;
913
914
915
916
917
918 to = (u8 *)end > (u8 *)tq->stat ? buf : sgl->sge;
919
920 for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
921 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
922 to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
923 to->addr[0] = cpu_to_be64(addr[i]);
924 to->addr[1] = cpu_to_be64(addr[++i]);
925 }
926 if (nfrags) {
927 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
928 to->len[1] = cpu_to_be32(0);
929 to->addr[0] = cpu_to_be64(addr[i + 1]);
930 }
931 if (unlikely((u8 *)end > (u8 *)tq->stat)) {
932 unsigned int part0 = (u8 *)tq->stat - (u8 *)sgl->sge, part1;
933
934 if (likely(part0))
935 memcpy(sgl->sge, buf, part0);
936 part1 = (u8 *)end - (u8 *)tq->stat;
937 memcpy(tq->desc, (u8 *)buf + part0, part1);
938 end = (void *)tq->desc + part1;
939 }
940 if ((uintptr_t)end & 8)
941 *end = 0;
942}
943
944
945
946
947
948
949
950
951
952static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq,
953 int n)
954{
955
956
957
958
959 WARN_ON((QID(tq->cntxt_id) | PIDX(n)) & DBPRIO(1));
960 wmb();
961 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
962 QID(tq->cntxt_id) | PIDX(n));
963}
964
965
966
967
968
969
970
971
972
973
974
975
976static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *tq,
977 void *pos)
978{
979 u64 *p;
980 int left = (void *)tq->stat - pos;
981
982 if (likely(skb->len <= left)) {
983 if (likely(!skb->data_len))
984 skb_copy_from_linear_data(skb, pos, skb->len);
985 else
986 skb_copy_bits(skb, 0, pos, skb->len);
987 pos += skb->len;
988 } else {
989 skb_copy_bits(skb, 0, pos, left);
990 skb_copy_bits(skb, left, tq->desc, skb->len - left);
991 pos = (void *)tq->desc + (skb->len - left);
992 }
993
994
995 p = PTR_ALIGN(pos, 8);
996 if ((uintptr_t)p & 8)
997 *p = 0;
998}
999
1000
1001
1002
1003
1004static u64 hwcsum(const struct sk_buff *skb)
1005{
1006 int csum_type;
1007 const struct iphdr *iph = ip_hdr(skb);
1008
1009 if (iph->version == 4) {
1010 if (iph->protocol == IPPROTO_TCP)
1011 csum_type = TX_CSUM_TCPIP;
1012 else if (iph->protocol == IPPROTO_UDP)
1013 csum_type = TX_CSUM_UDPIP;
1014 else {
1015nocsum:
1016
1017
1018
1019
1020 return TXPKT_L4CSUM_DIS;
1021 }
1022 } else {
1023
1024
1025
1026 const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph;
1027
1028 if (ip6h->nexthdr == IPPROTO_TCP)
1029 csum_type = TX_CSUM_TCPIP6;
1030 else if (ip6h->nexthdr == IPPROTO_UDP)
1031 csum_type = TX_CSUM_UDPIP6;
1032 else
1033 goto nocsum;
1034 }
1035
1036 if (likely(csum_type >= TX_CSUM_TCPIP))
1037 return TXPKT_CSUM_TYPE(csum_type) |
1038 TXPKT_IPHDR_LEN(skb_network_header_len(skb)) |
1039 TXPKT_ETHHDR_LEN(skb_network_offset(skb) - ETH_HLEN);
1040 else {
1041 int start = skb_transport_offset(skb);
1042
1043 return TXPKT_CSUM_TYPE(csum_type) |
1044 TXPKT_CSUM_START(start) |
1045 TXPKT_CSUM_LOC(start + skb->csum_offset);
1046 }
1047}
1048
1049
1050
1051
1052static void txq_stop(struct sge_eth_txq *txq)
1053{
1054 netif_tx_stop_queue(txq->txq);
1055 txq->q.stops++;
1056}
1057
1058
1059
1060
1061static inline void txq_advance(struct sge_txq *tq, unsigned int n)
1062{
1063 tq->in_use += n;
1064 tq->pidx += n;
1065 if (tq->pidx >= tq->size)
1066 tq->pidx -= tq->size;
1067}
1068
1069
1070
1071
1072
1073
1074
1075
1076int t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1077{
1078 u32 wr_mid;
1079 u64 cntrl, *end;
1080 int qidx, credits;
1081 unsigned int flits, ndesc;
1082 struct adapter *adapter;
1083 struct sge_eth_txq *txq;
1084 const struct port_info *pi;
1085 struct fw_eth_tx_pkt_vm_wr *wr;
1086 struct cpl_tx_pkt_core *cpl;
1087 const struct skb_shared_info *ssi;
1088 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1089 const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) +
1090 sizeof(wr->ethmacsrc) +
1091 sizeof(wr->ethtype) +
1092 sizeof(wr->vlantci));
1093
1094
1095
1096
1097
1098
1099
1100 if (unlikely(skb->len < fw_hdr_copy_len))
1101 goto out_free;
1102
1103
1104
1105
1106 pi = netdev_priv(dev);
1107 adapter = pi->adapter;
1108 qidx = skb_get_queue_mapping(skb);
1109 BUG_ON(qidx >= pi->nqsets);
1110 txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1111
1112
1113
1114
1115
1116 reclaim_completed_tx(adapter, &txq->q, true);
1117
1118
1119
1120
1121
1122
1123 flits = calc_tx_flits(skb);
1124 ndesc = flits_to_desc(flits);
1125 credits = txq_avail(&txq->q) - ndesc;
1126
1127 if (unlikely(credits < 0)) {
1128
1129
1130
1131
1132
1133
1134 txq_stop(txq);
1135 dev_err(adapter->pdev_dev,
1136 "%s: TX ring %u full while queue awake!\n",
1137 dev->name, qidx);
1138 return NETDEV_TX_BUSY;
1139 }
1140
1141 if (!is_eth_imm(skb) &&
1142 unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1143
1144
1145
1146
1147
1148 txq->mapping_err++;
1149 goto out_free;
1150 }
1151
1152 wr_mid = FW_WR_LEN16(DIV_ROUND_UP(flits, 2));
1153 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163 txq_stop(txq);
1164 wr_mid |= FW_WR_EQUEQ | FW_WR_EQUIQ;
1165 }
1166
1167
1168
1169
1170
1171
1172
1173 BUG_ON(DIV_ROUND_UP(ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1174 wr = (void *)&txq->q.desc[txq->q.pidx];
1175 wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1176 wr->r3[0] = cpu_to_be64(0);
1177 wr->r3[1] = cpu_to_be64(0);
1178 skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1179 end = (u64 *)wr + flits;
1180
1181
1182
1183
1184
1185
1186 ssi = skb_shinfo(skb);
1187 if (ssi->gso_size) {
1188 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1189 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1190 int l3hdr_len = skb_network_header_len(skb);
1191 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1192
1193 wr->op_immdlen =
1194 cpu_to_be32(FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
1195 FW_WR_IMMDLEN(sizeof(*lso) +
1196 sizeof(*cpl)));
1197
1198
1199
1200 lso->lso_ctrl =
1201 cpu_to_be32(LSO_OPCODE(CPL_TX_PKT_LSO) |
1202 LSO_FIRST_SLICE |
1203 LSO_LAST_SLICE |
1204 LSO_IPV6(v6) |
1205 LSO_ETHHDR_LEN(eth_xtra_len/4) |
1206 LSO_IPHDR_LEN(l3hdr_len/4) |
1207 LSO_TCPHDR_LEN(tcp_hdr(skb)->doff));
1208 lso->ipid_ofst = cpu_to_be16(0);
1209 lso->mss = cpu_to_be16(ssi->gso_size);
1210 lso->seqno_offset = cpu_to_be32(0);
1211 lso->len = cpu_to_be32(skb->len);
1212
1213
1214
1215
1216
1217 cpl = (void *)(lso + 1);
1218 cntrl = (TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1219 TXPKT_IPHDR_LEN(l3hdr_len) |
1220 TXPKT_ETHHDR_LEN(eth_xtra_len));
1221 txq->tso++;
1222 txq->tx_cso += ssi->gso_segs;
1223 } else {
1224 int len;
1225
1226 len = is_eth_imm(skb) ? skb->len + sizeof(*cpl) : sizeof(*cpl);
1227 wr->op_immdlen =
1228 cpu_to_be32(FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
1229 FW_WR_IMMDLEN(len));
1230
1231
1232
1233
1234
1235 cpl = (void *)(wr + 1);
1236 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1237 cntrl = hwcsum(skb) | TXPKT_IPCSUM_DIS;
1238 txq->tx_cso++;
1239 } else
1240 cntrl = TXPKT_L4CSUM_DIS | TXPKT_IPCSUM_DIS;
1241 }
1242
1243
1244
1245
1246
1247 if (vlan_tx_tag_present(skb)) {
1248 txq->vlan_ins++;
1249 cntrl |= TXPKT_VLAN_VLD | TXPKT_VLAN(vlan_tx_tag_get(skb));
1250 }
1251
1252
1253
1254
1255 cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE(CPL_TX_PKT_XT) |
1256 TXPKT_INTF(pi->port_id) |
1257 TXPKT_PF(0));
1258 cpl->pack = cpu_to_be16(0);
1259 cpl->len = cpu_to_be16(skb->len);
1260 cpl->ctrl1 = cpu_to_be64(cntrl);
1261
1262#ifdef T4_TRACE
1263 T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7],
1264 "eth_xmit: ndesc %u, credits %u, pidx %u, len %u, frags %u",
1265 ndesc, credits, txq->q.pidx, skb->len, ssi->nr_frags);
1266#endif
1267
1268
1269
1270
1271
1272 if (is_eth_imm(skb)) {
1273
1274
1275
1276
1277 inline_tx_skb(skb, &txq->q, cpl + 1);
1278 dev_kfree_skb(skb);
1279 } else {
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317 struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1318 struct sge_txq *tq = &txq->q;
1319 int last_desc;
1320
1321
1322
1323
1324
1325
1326
1327
1328 if (unlikely((void *)sgl == (void *)tq->stat)) {
1329 sgl = (void *)tq->desc;
1330 end = ((void *)tq->desc + ((void *)end - (void *)tq->stat));
1331 }
1332
1333 write_sgl(skb, tq, sgl, end, 0, addr);
1334 skb_orphan(skb);
1335
1336 last_desc = tq->pidx + ndesc - 1;
1337 if (last_desc >= tq->size)
1338 last_desc -= tq->size;
1339 tq->sdesc[last_desc].skb = skb;
1340 tq->sdesc[last_desc].sgl = sgl;
1341 }
1342
1343
1344
1345
1346
1347 txq_advance(&txq->q, ndesc);
1348 dev->trans_start = jiffies;
1349 ring_tx_db(adapter, &txq->q, ndesc);
1350 return NETDEV_TX_OK;
1351
1352out_free:
1353
1354
1355
1356
1357 dev_kfree_skb(skb);
1358 return NETDEV_TX_OK;
1359}
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370static inline void copy_frags(struct sk_buff *skb,
1371 const struct pkt_gl *gl,
1372 unsigned int offset)
1373{
1374 int i;
1375
1376
1377 __skb_fill_page_desc(skb, 0, gl->frags[0].page,
1378 gl->frags[0].offset + offset,
1379 gl->frags[0].size - offset);
1380 skb_shinfo(skb)->nr_frags = gl->nfrags;
1381 for (i = 1; i < gl->nfrags; i++)
1382 __skb_fill_page_desc(skb, i, gl->frags[i].page,
1383 gl->frags[i].offset,
1384 gl->frags[i].size);
1385
1386
1387 get_page(gl->frags[gl->nfrags - 1].page);
1388}
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399struct sk_buff *t4vf_pktgl_to_skb(const struct pkt_gl *gl,
1400 unsigned int skb_len, unsigned int pull_len)
1401{
1402 struct sk_buff *skb;
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415 if (gl->tot_len <= RX_COPY_THRES) {
1416
1417 skb = alloc_skb(gl->tot_len, GFP_ATOMIC);
1418 if (unlikely(!skb))
1419 goto out;
1420 __skb_put(skb, gl->tot_len);
1421 skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
1422 } else {
1423 skb = alloc_skb(skb_len, GFP_ATOMIC);
1424 if (unlikely(!skb))
1425 goto out;
1426 __skb_put(skb, pull_len);
1427 skb_copy_to_linear_data(skb, gl->va, pull_len);
1428
1429 copy_frags(skb, gl, pull_len);
1430 skb->len = gl->tot_len;
1431 skb->data_len = skb->len - pull_len;
1432 skb->truesize += skb->data_len;
1433 }
1434
1435out:
1436 return skb;
1437}
1438
1439
1440
1441
1442
1443
1444
1445
1446void t4vf_pktgl_free(const struct pkt_gl *gl)
1447{
1448 int frag;
1449
1450 frag = gl->nfrags - 1;
1451 while (frag--)
1452 put_page(gl->frags[frag].page);
1453}
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
1465 const struct cpl_rx_pkt *pkt)
1466{
1467 int ret;
1468 struct sk_buff *skb;
1469
1470 skb = napi_get_frags(&rxq->rspq.napi);
1471 if (unlikely(!skb)) {
1472 t4vf_pktgl_free(gl);
1473 rxq->stats.rx_drops++;
1474 return;
1475 }
1476
1477 copy_frags(skb, gl, PKTSHIFT);
1478 skb->len = gl->tot_len - PKTSHIFT;
1479 skb->data_len = skb->len;
1480 skb->truesize += skb->data_len;
1481 skb->ip_summed = CHECKSUM_UNNECESSARY;
1482 skb_record_rx_queue(skb, rxq->rspq.idx);
1483
1484 if (pkt->vlan_ex) {
1485 __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1486 be16_to_cpu(pkt->vlan));
1487 rxq->stats.vlan_ex++;
1488 }
1489 ret = napi_gro_frags(&rxq->rspq.napi);
1490
1491 if (ret == GRO_HELD)
1492 rxq->stats.lro_pkts++;
1493 else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
1494 rxq->stats.lro_merged++;
1495 rxq->stats.pkts++;
1496 rxq->stats.rx_cso++;
1497}
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
1508 const struct pkt_gl *gl)
1509{
1510 struct sk_buff *skb;
1511 const struct cpl_rx_pkt *pkt = (void *)rsp;
1512 bool csum_ok = pkt->csum_calc && !pkt->err_vec;
1513 struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1514
1515
1516
1517
1518
1519 if ((pkt->l2info & cpu_to_be32(RXF_TCP)) &&
1520 (rspq->netdev->features & NETIF_F_GRO) && csum_ok &&
1521 !pkt->ip_frag) {
1522 do_gro(rxq, gl, pkt);
1523 return 0;
1524 }
1525
1526
1527
1528
1529 skb = t4vf_pktgl_to_skb(gl, RX_SKB_LEN, RX_PULL_LEN);
1530 if (unlikely(!skb)) {
1531 t4vf_pktgl_free(gl);
1532 rxq->stats.rx_drops++;
1533 return 0;
1534 }
1535 __skb_pull(skb, PKTSHIFT);
1536 skb->protocol = eth_type_trans(skb, rspq->netdev);
1537 skb_record_rx_queue(skb, rspq->idx);
1538 rxq->stats.pkts++;
1539
1540 if (csum_ok && (rspq->netdev->features & NETIF_F_RXCSUM) &&
1541 !pkt->err_vec && (be32_to_cpu(pkt->l2info) & (RXF_UDP|RXF_TCP))) {
1542 if (!pkt->ip_frag)
1543 skb->ip_summed = CHECKSUM_UNNECESSARY;
1544 else {
1545 __sum16 c = (__force __sum16)pkt->csum;
1546 skb->csum = csum_unfold(c);
1547 skb->ip_summed = CHECKSUM_COMPLETE;
1548 }
1549 rxq->stats.rx_cso++;
1550 } else
1551 skb_checksum_none_assert(skb);
1552
1553 if (pkt->vlan_ex) {
1554 rxq->stats.vlan_ex++;
1555 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(pkt->vlan));
1556 }
1557
1558 netif_receive_skb(skb);
1559
1560 return 0;
1561}
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571static inline bool is_new_response(const struct rsp_ctrl *rc,
1572 const struct sge_rspq *rspq)
1573{
1574 return RSPD_GEN(rc->type_gen) == rspq->gen;
1575}
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597static void restore_rx_bufs(const struct pkt_gl *gl, struct sge_fl *fl,
1598 int frags)
1599{
1600 struct rx_sw_desc *sdesc;
1601
1602 while (frags--) {
1603 if (fl->cidx == 0)
1604 fl->cidx = fl->size - 1;
1605 else
1606 fl->cidx--;
1607 sdesc = &fl->sdesc[fl->cidx];
1608 sdesc->page = gl->frags[frags].page;
1609 sdesc->dma_addr |= RX_UNMAPPED_BUF;
1610 fl->avail++;
1611 }
1612}
1613
1614
1615
1616
1617
1618
1619
1620static inline void rspq_next(struct sge_rspq *rspq)
1621{
1622 rspq->cur_desc = (void *)rspq->cur_desc + rspq->iqe_len;
1623 if (unlikely(++rspq->cidx == rspq->size)) {
1624 rspq->cidx = 0;
1625 rspq->gen ^= 1;
1626 rspq->cur_desc = rspq->desc;
1627 }
1628}
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643int process_responses(struct sge_rspq *rspq, int budget)
1644{
1645 struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1646 int budget_left = budget;
1647
1648 while (likely(budget_left)) {
1649 int ret, rsp_type;
1650 const struct rsp_ctrl *rc;
1651
1652 rc = (void *)rspq->cur_desc + (rspq->iqe_len - sizeof(*rc));
1653 if (!is_new_response(rc, rspq))
1654 break;
1655
1656
1657
1658
1659
1660 rmb();
1661 rsp_type = RSPD_TYPE(rc->type_gen);
1662 if (likely(rsp_type == RSP_TYPE_FLBUF)) {
1663 struct page_frag *fp;
1664 struct pkt_gl gl;
1665 const struct rx_sw_desc *sdesc;
1666 u32 bufsz, frag;
1667 u32 len = be32_to_cpu(rc->pldbuflen_qid);
1668
1669
1670
1671
1672
1673 if (len & RSPD_NEWBUF) {
1674
1675
1676
1677
1678
1679 if (likely(rspq->offset > 0)) {
1680 free_rx_bufs(rspq->adapter, &rxq->fl,
1681 1);
1682 rspq->offset = 0;
1683 }
1684 len = RSPD_LEN(len);
1685 }
1686 gl.tot_len = len;
1687
1688
1689
1690
1691 for (frag = 0, fp = gl.frags; ; frag++, fp++) {
1692 BUG_ON(frag >= MAX_SKB_FRAGS);
1693 BUG_ON(rxq->fl.avail == 0);
1694 sdesc = &rxq->fl.sdesc[rxq->fl.cidx];
1695 bufsz = get_buf_size(sdesc);
1696 fp->page = sdesc->page;
1697 fp->offset = rspq->offset;
1698 fp->size = min(bufsz, len);
1699 len -= fp->size;
1700 if (!len)
1701 break;
1702 unmap_rx_buf(rspq->adapter, &rxq->fl);
1703 }
1704 gl.nfrags = frag+1;
1705
1706
1707
1708
1709
1710
1711 dma_sync_single_for_cpu(rspq->adapter->pdev_dev,
1712 get_buf_addr(sdesc),
1713 fp->size, DMA_FROM_DEVICE);
1714 gl.va = (page_address(gl.frags[0].page) +
1715 gl.frags[0].offset);
1716 prefetch(gl.va);
1717
1718
1719
1720
1721
1722 ret = rspq->handler(rspq, rspq->cur_desc, &gl);
1723 if (likely(ret == 0))
1724 rspq->offset += ALIGN(fp->size, FL_ALIGN);
1725 else
1726 restore_rx_bufs(&gl, &rxq->fl, frag);
1727 } else if (likely(rsp_type == RSP_TYPE_CPL)) {
1728 ret = rspq->handler(rspq, rspq->cur_desc, NULL);
1729 } else {
1730 WARN_ON(rsp_type > RSP_TYPE_CPL);
1731 ret = 0;
1732 }
1733
1734 if (unlikely(ret)) {
1735
1736
1737
1738
1739
1740 const int NOMEM_TIMER_IDX = SGE_NTIMERS-1;
1741 rspq->next_intr_params =
1742 QINTR_TIMER_IDX(NOMEM_TIMER_IDX);
1743 break;
1744 }
1745
1746 rspq_next(rspq);
1747 budget_left--;
1748 }
1749
1750
1751
1752
1753
1754
1755 if (rspq->offset >= 0 &&
1756 rxq->fl.size - rxq->fl.avail >= 2*FL_PER_EQ_UNIT)
1757 __refill_fl(rspq->adapter, &rxq->fl);
1758 return budget - budget_left;
1759}
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772static int napi_rx_handler(struct napi_struct *napi, int budget)
1773{
1774 unsigned int intr_params;
1775 struct sge_rspq *rspq = container_of(napi, struct sge_rspq, napi);
1776 int work_done = process_responses(rspq, budget);
1777
1778 if (likely(work_done < budget)) {
1779 napi_complete(napi);
1780 intr_params = rspq->next_intr_params;
1781 rspq->next_intr_params = rspq->intr_params;
1782 } else
1783 intr_params = QINTR_TIMER_IDX(SGE_TIMER_UPD_CIDX);
1784
1785 if (unlikely(work_done == 0))
1786 rspq->unhandled_irqs++;
1787
1788 t4_write_reg(rspq->adapter,
1789 T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
1790 CIDXINC(work_done) |
1791 INGRESSQID((u32)rspq->cntxt_id) |
1792 SEINTARM(intr_params));
1793 return work_done;
1794}
1795
1796
1797
1798
1799
1800irqreturn_t t4vf_sge_intr_msix(int irq, void *cookie)
1801{
1802 struct sge_rspq *rspq = cookie;
1803
1804 napi_schedule(&rspq->napi);
1805 return IRQ_HANDLED;
1806}
1807
1808
1809
1810
1811
1812static unsigned int process_intrq(struct adapter *adapter)
1813{
1814 struct sge *s = &adapter->sge;
1815 struct sge_rspq *intrq = &s->intrq;
1816 unsigned int work_done;
1817
1818 spin_lock(&adapter->sge.intrq_lock);
1819 for (work_done = 0; ; work_done++) {
1820 const struct rsp_ctrl *rc;
1821 unsigned int qid, iq_idx;
1822 struct sge_rspq *rspq;
1823
1824
1825
1826
1827
1828 rc = (void *)intrq->cur_desc + (intrq->iqe_len - sizeof(*rc));
1829 if (!is_new_response(rc, intrq))
1830 break;
1831
1832
1833
1834
1835
1836
1837 rmb();
1838 if (unlikely(RSPD_TYPE(rc->type_gen) != RSP_TYPE_INTR)) {
1839 dev_err(adapter->pdev_dev,
1840 "Unexpected INTRQ response type %d\n",
1841 RSPD_TYPE(rc->type_gen));
1842 continue;
1843 }
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853 qid = RSPD_QID(be32_to_cpu(rc->pldbuflen_qid));
1854 iq_idx = IQ_IDX(s, qid);
1855 if (unlikely(iq_idx >= MAX_INGQ)) {
1856 dev_err(adapter->pdev_dev,
1857 "Ingress QID %d out of range\n", qid);
1858 continue;
1859 }
1860 rspq = s->ingr_map[iq_idx];
1861 if (unlikely(rspq == NULL)) {
1862 dev_err(adapter->pdev_dev,
1863 "Ingress QID %d RSPQ=NULL\n", qid);
1864 continue;
1865 }
1866 if (unlikely(rspq->abs_id != qid)) {
1867 dev_err(adapter->pdev_dev,
1868 "Ingress QID %d refers to RSPQ %d\n",
1869 qid, rspq->abs_id);
1870 continue;
1871 }
1872
1873
1874
1875
1876
1877
1878 napi_schedule(&rspq->napi);
1879 rspq_next(intrq);
1880 }
1881
1882 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
1883 CIDXINC(work_done) |
1884 INGRESSQID(intrq->cntxt_id) |
1885 SEINTARM(intrq->intr_params));
1886
1887 spin_unlock(&adapter->sge.intrq_lock);
1888
1889 return work_done;
1890}
1891
1892
1893
1894
1895
1896irqreturn_t t4vf_intr_msi(int irq, void *cookie)
1897{
1898 struct adapter *adapter = cookie;
1899
1900 process_intrq(adapter);
1901 return IRQ_HANDLED;
1902}
1903
1904
1905
1906
1907
1908
1909
1910
1911irq_handler_t t4vf_intr_handler(struct adapter *adapter)
1912{
1913 BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
1914 if (adapter->flags & USING_MSIX)
1915 return t4vf_sge_intr_msix;
1916 else
1917 return t4vf_intr_msi;
1918}
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931static void sge_rx_timer_cb(unsigned long data)
1932{
1933 struct adapter *adapter = (struct adapter *)data;
1934 struct sge *s = &adapter->sge;
1935 unsigned int i;
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945 for (i = 0; i < ARRAY_SIZE(s->starving_fl); i++) {
1946 unsigned long m;
1947
1948 for (m = s->starving_fl[i]; m; m &= m - 1) {
1949 unsigned int id = __ffs(m) + i * BITS_PER_LONG;
1950 struct sge_fl *fl = s->egr_map[id];
1951
1952 clear_bit(id, s->starving_fl);
1953 smp_mb__after_clear_bit();
1954
1955
1956
1957
1958
1959
1960
1961 if (fl_starving(fl)) {
1962 struct sge_eth_rxq *rxq;
1963
1964 rxq = container_of(fl, struct sge_eth_rxq, fl);
1965 if (napi_reschedule(&rxq->rspq.napi))
1966 fl->starving++;
1967 else
1968 set_bit(id, s->starving_fl);
1969 }
1970 }
1971 }
1972
1973
1974
1975
1976 mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
1977}
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990static void sge_tx_timer_cb(unsigned long data)
1991{
1992 struct adapter *adapter = (struct adapter *)data;
1993 struct sge *s = &adapter->sge;
1994 unsigned int i, budget;
1995
1996 budget = MAX_TIMER_TX_RECLAIM;
1997 i = s->ethtxq_rover;
1998 do {
1999 struct sge_eth_txq *txq = &s->ethtxq[i];
2000
2001 if (reclaimable(&txq->q) && __netif_tx_trylock(txq->txq)) {
2002 int avail = reclaimable(&txq->q);
2003
2004 if (avail > budget)
2005 avail = budget;
2006
2007 free_tx_desc(adapter, &txq->q, avail, true);
2008 txq->q.in_use -= avail;
2009 __netif_tx_unlock(txq->txq);
2010
2011 budget -= avail;
2012 if (!budget)
2013 break;
2014 }
2015
2016 i++;
2017 if (i >= s->ethqsets)
2018 i = 0;
2019 } while (i != s->ethtxq_rover);
2020 s->ethtxq_rover = i;
2021
2022
2023
2024
2025
2026
2027 mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
2028}
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
2041 bool iqasynch, struct net_device *dev,
2042 int intr_dest,
2043 struct sge_fl *fl, rspq_handler_t hnd)
2044{
2045 struct port_info *pi = netdev_priv(dev);
2046 struct fw_iq_cmd cmd, rpl;
2047 int ret, iqandst, flsz = 0;
2048
2049
2050
2051
2052
2053
2054
2055
2056 if ((adapter->flags & USING_MSI) && rspq != &adapter->sge.intrq) {
2057 iqandst = SGE_INTRDST_IQ;
2058 intr_dest = adapter->sge.intrq.abs_id;
2059 } else
2060 iqandst = SGE_INTRDST_PCI;
2061
2062
2063
2064
2065
2066
2067
2068 rspq->size = roundup(rspq->size, 16);
2069 rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len,
2070 0, &rspq->phys_addr, NULL, 0);
2071 if (!rspq->desc)
2072 return -ENOMEM;
2073
2074
2075
2076
2077
2078
2079
2080
2081 memset(&cmd, 0, sizeof(cmd));
2082 cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP(FW_IQ_CMD) |
2083 FW_CMD_REQUEST |
2084 FW_CMD_WRITE |
2085 FW_CMD_EXEC);
2086 cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_ALLOC |
2087 FW_IQ_CMD_IQSTART(1) |
2088 FW_LEN16(cmd));
2089 cmd.type_to_iqandstindex =
2090 cpu_to_be32(FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
2091 FW_IQ_CMD_IQASYNCH(iqasynch) |
2092 FW_IQ_CMD_VIID(pi->viid) |
2093 FW_IQ_CMD_IQANDST(iqandst) |
2094 FW_IQ_CMD_IQANUS(1) |
2095 FW_IQ_CMD_IQANUD(SGE_UPDATEDEL_INTR) |
2096 FW_IQ_CMD_IQANDSTINDEX(intr_dest));
2097 cmd.iqdroprss_to_iqesize =
2098 cpu_to_be16(FW_IQ_CMD_IQPCIECH(pi->port_id) |
2099 FW_IQ_CMD_IQGTSMODE |
2100 FW_IQ_CMD_IQINTCNTTHRESH(rspq->pktcnt_idx) |
2101 FW_IQ_CMD_IQESIZE(ilog2(rspq->iqe_len) - 4));
2102 cmd.iqsize = cpu_to_be16(rspq->size);
2103 cmd.iqaddr = cpu_to_be64(rspq->phys_addr);
2104
2105 if (fl) {
2106
2107
2108
2109
2110
2111
2112 fl->size = roundup(fl->size, FL_PER_EQ_UNIT);
2113 fl->desc = alloc_ring(adapter->pdev_dev, fl->size,
2114 sizeof(__be64), sizeof(struct rx_sw_desc),
2115 &fl->addr, &fl->sdesc, STAT_LEN);
2116 if (!fl->desc) {
2117 ret = -ENOMEM;
2118 goto err;
2119 }
2120
2121
2122
2123
2124
2125
2126 flsz = (fl->size / FL_PER_EQ_UNIT +
2127 STAT_LEN / EQ_UNIT);
2128
2129
2130
2131
2132
2133 cmd.iqns_to_fl0congen =
2134 cpu_to_be32(
2135 FW_IQ_CMD_FL0HOSTFCMODE(SGE_HOSTFCMODE_NONE) |
2136 FW_IQ_CMD_FL0PACKEN(1) |
2137 FW_IQ_CMD_FL0PADEN(1));
2138 cmd.fl0dcaen_to_fl0cidxfthresh =
2139 cpu_to_be16(
2140 FW_IQ_CMD_FL0FBMIN(SGE_FETCHBURSTMIN_64B) |
2141 FW_IQ_CMD_FL0FBMAX(SGE_FETCHBURSTMAX_512B));
2142 cmd.fl0size = cpu_to_be16(flsz);
2143 cmd.fl0addr = cpu_to_be64(fl->addr);
2144 }
2145
2146
2147
2148
2149
2150 ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2151 if (ret)
2152 goto err;
2153
2154 netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64);
2155 rspq->cur_desc = rspq->desc;
2156 rspq->cidx = 0;
2157 rspq->gen = 1;
2158 rspq->next_intr_params = rspq->intr_params;
2159 rspq->cntxt_id = be16_to_cpu(rpl.iqid);
2160 rspq->abs_id = be16_to_cpu(rpl.physiqid);
2161 rspq->size--;
2162 rspq->adapter = adapter;
2163 rspq->netdev = dev;
2164 rspq->handler = hnd;
2165
2166
2167 rspq->offset = fl ? 0 : -1;
2168
2169 if (fl) {
2170 fl->cntxt_id = be16_to_cpu(rpl.fl0id);
2171 fl->avail = 0;
2172 fl->pend_cred = 0;
2173 fl->pidx = 0;
2174 fl->cidx = 0;
2175 fl->alloc_failed = 0;
2176 fl->large_alloc_failed = 0;
2177 fl->starving = 0;
2178 refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL);
2179 }
2180
2181 return 0;
2182
2183err:
2184
2185
2186
2187
2188 if (rspq->desc) {
2189 dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len,
2190 rspq->desc, rspq->phys_addr);
2191 rspq->desc = NULL;
2192 }
2193 if (fl && fl->desc) {
2194 kfree(fl->sdesc);
2195 fl->sdesc = NULL;
2196 dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT,
2197 fl->desc, fl->addr);
2198 fl->desc = NULL;
2199 }
2200 return ret;
2201}
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq,
2212 struct net_device *dev, struct netdev_queue *devq,
2213 unsigned int iqid)
2214{
2215 int ret, nentries;
2216 struct fw_eq_eth_cmd cmd, rpl;
2217 struct port_info *pi = netdev_priv(dev);
2218
2219
2220
2221
2222
2223 nentries = txq->q.size + STAT_LEN / sizeof(struct tx_desc);
2224
2225
2226
2227
2228
2229 txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size,
2230 sizeof(struct tx_desc),
2231 sizeof(struct tx_sw_desc),
2232 &txq->q.phys_addr, &txq->q.sdesc, STAT_LEN);
2233 if (!txq->q.desc)
2234 return -ENOMEM;
2235
2236
2237
2238
2239
2240
2241
2242
2243 memset(&cmd, 0, sizeof(cmd));
2244 cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP(FW_EQ_ETH_CMD) |
2245 FW_CMD_REQUEST |
2246 FW_CMD_WRITE |
2247 FW_CMD_EXEC);
2248 cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_ALLOC |
2249 FW_EQ_ETH_CMD_EQSTART |
2250 FW_LEN16(cmd));
2251 cmd.viid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_VIID(pi->viid));
2252 cmd.fetchszm_to_iqid =
2253 cpu_to_be32(FW_EQ_ETH_CMD_HOSTFCMODE(SGE_HOSTFCMODE_STPG) |
2254 FW_EQ_ETH_CMD_PCIECHN(pi->port_id) |
2255 FW_EQ_ETH_CMD_IQID(iqid));
2256 cmd.dcaen_to_eqsize =
2257 cpu_to_be32(FW_EQ_ETH_CMD_FBMIN(SGE_FETCHBURSTMIN_64B) |
2258 FW_EQ_ETH_CMD_FBMAX(SGE_FETCHBURSTMAX_512B) |
2259 FW_EQ_ETH_CMD_CIDXFTHRESH(SGE_CIDXFLUSHTHRESH_32) |
2260 FW_EQ_ETH_CMD_EQSIZE(nentries));
2261 cmd.eqaddr = cpu_to_be64(txq->q.phys_addr);
2262
2263
2264
2265
2266
2267 ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2268 if (ret) {
2269
2270
2271
2272
2273 kfree(txq->q.sdesc);
2274 txq->q.sdesc = NULL;
2275 dma_free_coherent(adapter->pdev_dev,
2276 nentries * sizeof(struct tx_desc),
2277 txq->q.desc, txq->q.phys_addr);
2278 txq->q.desc = NULL;
2279 return ret;
2280 }
2281
2282 txq->q.in_use = 0;
2283 txq->q.cidx = 0;
2284 txq->q.pidx = 0;
2285 txq->q.stat = (void *)&txq->q.desc[txq->q.size];
2286 txq->q.cntxt_id = FW_EQ_ETH_CMD_EQID_GET(be32_to_cpu(rpl.eqid_pkd));
2287 txq->q.abs_id =
2288 FW_EQ_ETH_CMD_PHYSEQID_GET(be32_to_cpu(rpl.physeqid_pkd));
2289 txq->txq = devq;
2290 txq->tso = 0;
2291 txq->tx_cso = 0;
2292 txq->vlan_ins = 0;
2293 txq->q.stops = 0;
2294 txq->q.restarts = 0;
2295 txq->mapping_err = 0;
2296 return 0;
2297}
2298
2299
2300
2301
2302static void free_txq(struct adapter *adapter, struct sge_txq *tq)
2303{
2304 dma_free_coherent(adapter->pdev_dev,
2305 tq->size * sizeof(*tq->desc) + STAT_LEN,
2306 tq->desc, tq->phys_addr);
2307 tq->cntxt_id = 0;
2308 tq->sdesc = NULL;
2309 tq->desc = NULL;
2310}
2311
2312
2313
2314
2315
2316static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq,
2317 struct sge_fl *fl)
2318{
2319 unsigned int flid = fl ? fl->cntxt_id : 0xffff;
2320
2321 t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP,
2322 rspq->cntxt_id, flid, 0xffff);
2323 dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len,
2324 rspq->desc, rspq->phys_addr);
2325 netif_napi_del(&rspq->napi);
2326 rspq->netdev = NULL;
2327 rspq->cntxt_id = 0;
2328 rspq->abs_id = 0;
2329 rspq->desc = NULL;
2330
2331 if (fl) {
2332 free_rx_bufs(adapter, fl, fl->avail);
2333 dma_free_coherent(adapter->pdev_dev,
2334 fl->size * sizeof(*fl->desc) + STAT_LEN,
2335 fl->desc, fl->addr);
2336 kfree(fl->sdesc);
2337 fl->sdesc = NULL;
2338 fl->cntxt_id = 0;
2339 fl->desc = NULL;
2340 }
2341}
2342
2343
2344
2345
2346
2347
2348
2349void t4vf_free_sge_resources(struct adapter *adapter)
2350{
2351 struct sge *s = &adapter->sge;
2352 struct sge_eth_rxq *rxq = s->ethrxq;
2353 struct sge_eth_txq *txq = s->ethtxq;
2354 struct sge_rspq *evtq = &s->fw_evtq;
2355 struct sge_rspq *intrq = &s->intrq;
2356 int qs;
2357
2358 for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) {
2359 if (rxq->rspq.desc)
2360 free_rspq_fl(adapter, &rxq->rspq, &rxq->fl);
2361 if (txq->q.desc) {
2362 t4vf_eth_eq_free(adapter, txq->q.cntxt_id);
2363 free_tx_desc(adapter, &txq->q, txq->q.in_use, true);
2364 kfree(txq->q.sdesc);
2365 free_txq(adapter, &txq->q);
2366 }
2367 }
2368 if (evtq->desc)
2369 free_rspq_fl(adapter, evtq, NULL);
2370 if (intrq->desc)
2371 free_rspq_fl(adapter, intrq, NULL);
2372}
2373
2374
2375
2376
2377
2378
2379
2380void t4vf_sge_start(struct adapter *adapter)
2381{
2382 adapter->sge.ethtxq_rover = 0;
2383 mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
2384 mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
2385}
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395void t4vf_sge_stop(struct adapter *adapter)
2396{
2397 struct sge *s = &adapter->sge;
2398
2399 if (s->rx_timer.function)
2400 del_timer_sync(&s->rx_timer);
2401 if (s->tx_timer.function)
2402 del_timer_sync(&s->tx_timer);
2403}
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414int t4vf_sge_init(struct adapter *adapter)
2415{
2416 struct sge_params *sge_params = &adapter->params.sge;
2417 u32 fl0 = sge_params->sge_fl_buffer_size[0];
2418 u32 fl1 = sge_params->sge_fl_buffer_size[1];
2419 struct sge *s = &adapter->sge;
2420
2421
2422
2423
2424
2425
2426 if (fl0 != PAGE_SIZE || (fl1 != 0 && fl1 <= fl0)) {
2427 dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",
2428 fl0, fl1);
2429 return -EINVAL;
2430 }
2431 if ((sge_params->sge_control & RXPKTCPLMODE_MASK) == 0) {
2432 dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");
2433 return -EINVAL;
2434 }
2435
2436
2437
2438
2439 if (fl1)
2440 FL_PG_ORDER = ilog2(fl1) - PAGE_SHIFT;
2441 STAT_LEN = ((sge_params->sge_control & EGRSTATUSPAGESIZE_MASK)
2442 ? 128 : 64);
2443 PKTSHIFT = PKTSHIFT_GET(sge_params->sge_control);
2444 FL_ALIGN = 1 << (INGPADBOUNDARY_GET(sge_params->sge_control) +
2445 SGE_INGPADBOUNDARY_SHIFT);
2446
2447
2448
2449
2450 setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adapter);
2451 setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adapter);
2452
2453
2454
2455
2456 spin_lock_init(&s->intrq_lock);
2457
2458 return 0;
2459}
2460