1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#include <linux/skbuff.h>
36#include <linux/netdevice.h>
37#include <linux/etherdevice.h>
38#include <linux/if_vlan.h>
39#include <linux/ip.h>
40#include <linux/dma-mapping.h>
41#include <linux/jiffies.h>
42#include <linux/prefetch.h>
43#include <linux/export.h>
44#include <net/xfrm.h>
45#include <net/ipv6.h>
46#include <net/tcp.h>
47#include <net/busy_poll.h>
48#include "cxgb4.h"
49#include "t4_regs.h"
50#include "t4_values.h"
51#include "t4_msg.h"
52#include "t4fw_api.h"
53#include "cxgb4_ptp.h"
54#include "cxgb4_uld.h"
55
56
57
58
59
60#if PAGE_SHIFT >= 16
61# define FL_PG_ORDER 0
62#else
63# define FL_PG_ORDER (16 - PAGE_SHIFT)
64#endif
65
66
67#define RX_COPY_THRES 256
68#define RX_PULL_LEN 128
69
70
71
72
73
74#define RX_PKT_SKB_LEN 512
75
76
77
78
79
80
81
82#define MAX_TX_RECLAIM 16
83
84
85
86
87
88#define MAX_RX_REFILL 16U
89
90
91
92
93
94#define RX_QCHECK_PERIOD (HZ / 2)
95
96
97
98
99#define TX_QCHECK_PERIOD (HZ / 2)
100
101
102
103
104#define MAX_TIMER_TX_RECLAIM 100
105
106
107
108
109#define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
110
111
112
113
114
115#define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
116
117
118
119
120
121#define MAX_IMM_TX_PKT_LEN 256
122
123
124
125
126#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
127
128struct rx_sw_desc {
129 struct page *page;
130 dma_addr_t dma_addr;
131};
132
133
134
135
136
137
138
139#define FL_MTU_SMALL 1500
140#define FL_MTU_LARGE 9000
141
142static inline unsigned int fl_mtu_bufsize(struct adapter *adapter,
143 unsigned int mtu)
144{
145 struct sge *s = &adapter->sge;
146
147 return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align);
148}
149
150#define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
151#define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
152
153
154
155
156
157
158
159
160
161
162
163enum {
164 RX_BUF_FLAGS = 0x1f,
165 RX_BUF_SIZE = 0x0f,
166 RX_UNMAPPED_BUF = 0x10,
167
168
169
170
171
172
173
174
175 RX_SMALL_PG_BUF = 0x0,
176 RX_LARGE_PG_BUF = 0x1,
177
178 RX_SMALL_MTU_BUF = 0x2,
179 RX_LARGE_MTU_BUF = 0x3,
180};
181
182static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5};
183#define MIN_NAPI_WORK 1
184
185static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d)
186{
187 return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS;
188}
189
190static inline bool is_buf_mapped(const struct rx_sw_desc *d)
191{
192 return !(d->dma_addr & RX_UNMAPPED_BUF);
193}
194
195
196
197
198
199
200
201
202static inline unsigned int txq_avail(const struct sge_txq *q)
203{
204 return q->size - 1 - q->in_use;
205}
206
207
208
209
210
211
212
213
214
215static inline unsigned int fl_cap(const struct sge_fl *fl)
216{
217 return fl->size - 8;
218}
219
220
221
222
223
224
225
226
227
228
229static inline bool fl_starving(const struct adapter *adapter,
230 const struct sge_fl *fl)
231{
232 const struct sge *s = &adapter->sge;
233
234 return fl->avail - fl->pend_cred <= s->fl_starve_thres;
235}
236
237int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
238 dma_addr_t *addr)
239{
240 const skb_frag_t *fp, *end;
241 const struct skb_shared_info *si;
242
243 *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
244 if (dma_mapping_error(dev, *addr))
245 goto out_err;
246
247 si = skb_shinfo(skb);
248 end = &si->frags[si->nr_frags];
249
250 for (fp = si->frags; fp < end; fp++) {
251 *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
252 DMA_TO_DEVICE);
253 if (dma_mapping_error(dev, *addr))
254 goto unwind;
255 }
256 return 0;
257
258unwind:
259 while (fp-- > si->frags)
260 dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
261
262 dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
263out_err:
264 return -ENOMEM;
265}
266EXPORT_SYMBOL(cxgb4_map_skb);
267
268#ifdef CONFIG_NEED_DMA_MAP_STATE
269static void unmap_skb(struct device *dev, const struct sk_buff *skb,
270 const dma_addr_t *addr)
271{
272 const skb_frag_t *fp, *end;
273 const struct skb_shared_info *si;
274
275 dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE);
276
277 si = skb_shinfo(skb);
278 end = &si->frags[si->nr_frags];
279 for (fp = si->frags; fp < end; fp++)
280 dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE);
281}
282
283
284
285
286
287
288
289
290
291static void deferred_unmap_destructor(struct sk_buff *skb)
292{
293 unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head);
294}
295#endif
296
297static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
298 const struct ulptx_sgl *sgl, const struct sge_txq *q)
299{
300 const struct ulptx_sge_pair *p;
301 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
302
303 if (likely(skb_headlen(skb)))
304 dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
305 DMA_TO_DEVICE);
306 else {
307 dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
308 DMA_TO_DEVICE);
309 nfrags--;
310 }
311
312
313
314
315
316 for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
317 if (likely((u8 *)(p + 1) <= (u8 *)q->stat)) {
318unmap: dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
319 ntohl(p->len[0]), DMA_TO_DEVICE);
320 dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
321 ntohl(p->len[1]), DMA_TO_DEVICE);
322 p++;
323 } else if ((u8 *)p == (u8 *)q->stat) {
324 p = (const struct ulptx_sge_pair *)q->desc;
325 goto unmap;
326 } else if ((u8 *)p + 8 == (u8 *)q->stat) {
327 const __be64 *addr = (const __be64 *)q->desc;
328
329 dma_unmap_page(dev, be64_to_cpu(addr[0]),
330 ntohl(p->len[0]), DMA_TO_DEVICE);
331 dma_unmap_page(dev, be64_to_cpu(addr[1]),
332 ntohl(p->len[1]), DMA_TO_DEVICE);
333 p = (const struct ulptx_sge_pair *)&addr[2];
334 } else {
335 const __be64 *addr = (const __be64 *)q->desc;
336
337 dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
338 ntohl(p->len[0]), DMA_TO_DEVICE);
339 dma_unmap_page(dev, be64_to_cpu(addr[0]),
340 ntohl(p->len[1]), DMA_TO_DEVICE);
341 p = (const struct ulptx_sge_pair *)&addr[1];
342 }
343 }
344 if (nfrags) {
345 __be64 addr;
346
347 if ((u8 *)p == (u8 *)q->stat)
348 p = (const struct ulptx_sge_pair *)q->desc;
349 addr = (u8 *)p + 16 <= (u8 *)q->stat ? p->addr[0] :
350 *(const __be64 *)q->desc;
351 dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]),
352 DMA_TO_DEVICE);
353 }
354}
355
356
357
358
359
360
361
362
363
364
365
366void free_tx_desc(struct adapter *adap, struct sge_txq *q,
367 unsigned int n, bool unmap)
368{
369 struct tx_sw_desc *d;
370 unsigned int cidx = q->cidx;
371 struct device *dev = adap->pdev_dev;
372
373 d = &q->sdesc[cidx];
374 while (n--) {
375 if (d->skb) {
376 if (unmap)
377 unmap_sgl(dev, d->skb, d->sgl, q);
378 dev_consume_skb_any(d->skb);
379 d->skb = NULL;
380 }
381 ++d;
382 if (++cidx == q->size) {
383 cidx = 0;
384 d = q->sdesc;
385 }
386 }
387 q->cidx = cidx;
388}
389
390
391
392
393static inline int reclaimable(const struct sge_txq *q)
394{
395 int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
396 hw_cidx -= q->cidx;
397 return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx;
398}
399
400
401
402
403
404
405
406
407
408
409
410inline void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
411 bool unmap)
412{
413 int avail = reclaimable(q);
414
415 if (avail) {
416
417
418
419
420 if (avail > MAX_TX_RECLAIM)
421 avail = MAX_TX_RECLAIM;
422
423 free_tx_desc(adap, q, avail, unmap);
424 q->in_use -= avail;
425 }
426}
427EXPORT_SYMBOL(cxgb4_reclaim_completed_tx);
428
429static inline int get_buf_size(struct adapter *adapter,
430 const struct rx_sw_desc *d)
431{
432 struct sge *s = &adapter->sge;
433 unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
434 int buf_size;
435
436 switch (rx_buf_size_idx) {
437 case RX_SMALL_PG_BUF:
438 buf_size = PAGE_SIZE;
439 break;
440
441 case RX_LARGE_PG_BUF:
442 buf_size = PAGE_SIZE << s->fl_pg_order;
443 break;
444
445 case RX_SMALL_MTU_BUF:
446 buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
447 break;
448
449 case RX_LARGE_MTU_BUF:
450 buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
451 break;
452
453 default:
454 BUG_ON(1);
455 }
456
457 return buf_size;
458}
459
460
461
462
463
464
465
466
467
468
469static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n)
470{
471 while (n--) {
472 struct rx_sw_desc *d = &q->sdesc[q->cidx];
473
474 if (is_buf_mapped(d))
475 dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
476 get_buf_size(adap, d),
477 PCI_DMA_FROMDEVICE);
478 put_page(d->page);
479 d->page = NULL;
480 if (++q->cidx == q->size)
481 q->cidx = 0;
482 q->avail--;
483 }
484}
485
486
487
488
489
490
491
492
493
494
495
496
497static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q)
498{
499 struct rx_sw_desc *d = &q->sdesc[q->cidx];
500
501 if (is_buf_mapped(d))
502 dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
503 get_buf_size(adap, d), PCI_DMA_FROMDEVICE);
504 d->page = NULL;
505 if (++q->cidx == q->size)
506 q->cidx = 0;
507 q->avail--;
508}
509
510static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
511{
512 if (q->pend_cred >= 8) {
513 u32 val = adap->params.arch.sge_fl_db;
514
515 if (is_t4(adap->params.chip))
516 val |= PIDX_V(q->pend_cred / 8);
517 else
518 val |= PIDX_T5_V(q->pend_cred / 8);
519
520
521
522
523 wmb();
524
525
526
527
528
529 if (unlikely(q->bar2_addr == NULL)) {
530 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
531 val | QID_V(q->cntxt_id));
532 } else {
533 writel(val | QID_V(q->bar2_qid),
534 q->bar2_addr + SGE_UDB_KDOORBELL);
535
536
537
538
539 wmb();
540 }
541 q->pend_cred &= 7;
542 }
543}
544
545static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg,
546 dma_addr_t mapping)
547{
548 sd->page = pg;
549 sd->dma_addr = mapping;
550}
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
567 gfp_t gfp)
568{
569 struct sge *s = &adap->sge;
570 struct page *pg;
571 dma_addr_t mapping;
572 unsigned int cred = q->avail;
573 __be64 *d = &q->desc[q->pidx];
574 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
575 int node;
576
577#ifdef CONFIG_DEBUG_FS
578 if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl))
579 goto out;
580#endif
581
582 gfp |= __GFP_NOWARN | __GFP_COLD;
583 node = dev_to_node(adap->pdev_dev);
584
585 if (s->fl_pg_order == 0)
586 goto alloc_small_pages;
587
588
589
590
591 while (n) {
592 pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order);
593 if (unlikely(!pg)) {
594 q->large_alloc_failed++;
595 break;
596 }
597
598 mapping = dma_map_page(adap->pdev_dev, pg, 0,
599 PAGE_SIZE << s->fl_pg_order,
600 PCI_DMA_FROMDEVICE);
601 if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
602 __free_pages(pg, s->fl_pg_order);
603 q->mapping_err++;
604 goto out;
605 }
606 mapping |= RX_LARGE_PG_BUF;
607 *d++ = cpu_to_be64(mapping);
608
609 set_rx_sw_desc(sd, pg, mapping);
610 sd++;
611
612 q->avail++;
613 if (++q->pidx == q->size) {
614 q->pidx = 0;
615 sd = q->sdesc;
616 d = q->desc;
617 }
618 n--;
619 }
620
621alloc_small_pages:
622 while (n--) {
623 pg = alloc_pages_node(node, gfp, 0);
624 if (unlikely(!pg)) {
625 q->alloc_failed++;
626 break;
627 }
628
629 mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE,
630 PCI_DMA_FROMDEVICE);
631 if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
632 put_page(pg);
633 q->mapping_err++;
634 goto out;
635 }
636 *d++ = cpu_to_be64(mapping);
637
638 set_rx_sw_desc(sd, pg, mapping);
639 sd++;
640
641 q->avail++;
642 if (++q->pidx == q->size) {
643 q->pidx = 0;
644 sd = q->sdesc;
645 d = q->desc;
646 }
647 }
648
649out: cred = q->avail - cred;
650 q->pend_cred += cred;
651 ring_fl_db(adap, q);
652
653 if (unlikely(fl_starving(adap, q))) {
654 smp_wmb();
655 q->low++;
656 set_bit(q->cntxt_id - adap->sge.egr_start,
657 adap->sge.starving_fl);
658 }
659
660 return cred;
661}
662
663static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
664{
665 refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail),
666 GFP_ATOMIC);
667}
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size,
689 size_t sw_size, dma_addr_t *phys, void *metadata,
690 size_t stat_size, int node)
691{
692 size_t len = nelem * elem_size + stat_size;
693 void *s = NULL;
694 void *p = dma_zalloc_coherent(dev, len, phys, GFP_KERNEL);
695
696 if (!p)
697 return NULL;
698 if (sw_size) {
699 s = kcalloc_node(sw_size, nelem, GFP_KERNEL, node);
700
701 if (!s) {
702 dma_free_coherent(dev, len, p, *phys);
703 return NULL;
704 }
705 }
706 if (metadata)
707 *(void **)metadata = s;
708 return p;
709}
710
711
712
713
714
715
716
717
718static inline unsigned int sgl_len(unsigned int n)
719{
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736 n--;
737 return (3 * n) / 2 + (n & 1) + 2;
738}
739
740
741
742
743
744
745
746
747static inline unsigned int flits_to_desc(unsigned int n)
748{
749 BUG_ON(n > SGE_MAX_WR_LEN / 8);
750 return DIV_ROUND_UP(n, 8);
751}
752
753
754
755
756
757
758
759
760static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver)
761{
762 int hdrlen = 0;
763
764 if (skb->encapsulation && skb_shinfo(skb)->gso_size &&
765 chip_ver > CHELSIO_T5) {
766 hdrlen = sizeof(struct cpl_tx_tnl_lso);
767 hdrlen += sizeof(struct cpl_tx_pkt_core);
768 } else {
769 hdrlen = skb_shinfo(skb)->gso_size ?
770 sizeof(struct cpl_tx_pkt_lso_core) : 0;
771 hdrlen += sizeof(struct cpl_tx_pkt);
772 }
773 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
774 return hdrlen;
775 return 0;
776}
777
778
779
780
781
782
783
784
785static inline unsigned int calc_tx_flits(const struct sk_buff *skb,
786 unsigned int chip_ver)
787{
788 unsigned int flits;
789 int hdrlen = is_eth_imm(skb, chip_ver);
790
791
792
793
794
795
796 if (hdrlen)
797 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
798
799
800
801
802
803
804
805
806
807 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
808 if (skb_shinfo(skb)->gso_size) {
809 if (skb->encapsulation && chip_ver > CHELSIO_T5)
810 hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
811 sizeof(struct cpl_tx_tnl_lso);
812 else
813 hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
814 sizeof(struct cpl_tx_pkt_lso_core);
815
816 hdrlen += sizeof(struct cpl_tx_pkt_core);
817 flits += (hdrlen / sizeof(__be64));
818 } else {
819 flits += (sizeof(struct fw_eth_tx_pkt_wr) +
820 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
821 }
822 return flits;
823}
824
825
826
827
828
829
830
831
832static inline unsigned int calc_tx_descs(const struct sk_buff *skb,
833 unsigned int chip_ver)
834{
835 return flits_to_desc(calc_tx_flits(skb, chip_ver));
836}
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
856 struct ulptx_sgl *sgl, u64 *end, unsigned int start,
857 const dma_addr_t *addr)
858{
859 unsigned int i, len;
860 struct ulptx_sge_pair *to;
861 const struct skb_shared_info *si = skb_shinfo(skb);
862 unsigned int nfrags = si->nr_frags;
863 struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
864
865 len = skb_headlen(skb) - start;
866 if (likely(len)) {
867 sgl->len0 = htonl(len);
868 sgl->addr0 = cpu_to_be64(addr[0] + start);
869 nfrags++;
870 } else {
871 sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
872 sgl->addr0 = cpu_to_be64(addr[1]);
873 }
874
875 sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
876 ULPTX_NSGE_V(nfrags));
877 if (likely(--nfrags == 0))
878 return;
879
880
881
882
883
884 to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge;
885
886 for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
887 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
888 to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
889 to->addr[0] = cpu_to_be64(addr[i]);
890 to->addr[1] = cpu_to_be64(addr[++i]);
891 }
892 if (nfrags) {
893 to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
894 to->len[1] = cpu_to_be32(0);
895 to->addr[0] = cpu_to_be64(addr[i + 1]);
896 }
897 if (unlikely((u8 *)end > (u8 *)q->stat)) {
898 unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1;
899
900 if (likely(part0))
901 memcpy(sgl->sge, buf, part0);
902 part1 = (u8 *)end - (u8 *)q->stat;
903 memcpy(q->desc, (u8 *)buf + part0, part1);
904 end = (void *)q->desc + part1;
905 }
906 if ((uintptr_t)end & 8)
907 *end = 0;
908}
909EXPORT_SYMBOL(cxgb4_write_sgl);
910
911
912
913
914
915static void cxgb_pio_copy(u64 __iomem *dst, u64 *src)
916{
917 int count = 8;
918
919 while (count) {
920 writeq(*src, dst);
921 src++;
922 dst++;
923 count--;
924 }
925}
926
927
928
929
930
931
932
933
934
935inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
936{
937
938
939
940 wmb();
941
942
943
944
945 if (unlikely(q->bar2_addr == NULL)) {
946 u32 val = PIDX_V(n);
947 unsigned long flags;
948
949
950
951
952 spin_lock_irqsave(&q->db_lock, flags);
953 if (!q->db_disabled)
954 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
955 QID_V(q->cntxt_id) | val);
956 else
957 q->db_pidx_inc += n;
958 q->db_pidx = q->pidx;
959 spin_unlock_irqrestore(&q->db_lock, flags);
960 } else {
961 u32 val = PIDX_T5_V(n);
962
963
964
965
966
967
968
969 WARN_ON(val & DBPRIO_F);
970
971
972
973
974
975 if (n == 1 && q->bar2_qid == 0) {
976 int index = (q->pidx
977 ? (q->pidx - 1)
978 : (q->size - 1));
979 u64 *wr = (u64 *)&q->desc[index];
980
981 cxgb_pio_copy((u64 __iomem *)
982 (q->bar2_addr + SGE_UDB_WCDOORBELL),
983 wr);
984 } else {
985 writel(val | QID_V(q->bar2_qid),
986 q->bar2_addr + SGE_UDB_KDOORBELL);
987}
988
989
990
991
992
993
994
995
996
997
998
999 wmb();
1000 }
1001}
1002EXPORT_SYMBOL(cxgb4_ring_tx_db);
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015void cxgb4_inline_tx_skb(const struct sk_buff *skb,
1016 const struct sge_txq *q, void *pos)
1017{
1018 int left = (void *)q->stat - pos;
1019 u64 *p;
1020
1021 if (likely(skb->len <= left)) {
1022 if (likely(!skb->data_len))
1023 skb_copy_from_linear_data(skb, pos, skb->len);
1024 else
1025 skb_copy_bits(skb, 0, pos, skb->len);
1026 pos += skb->len;
1027 } else {
1028 skb_copy_bits(skb, 0, pos, left);
1029 skb_copy_bits(skb, left, q->desc, skb->len - left);
1030 pos = (void *)q->desc + (skb->len - left);
1031 }
1032
1033
1034 p = PTR_ALIGN(pos, 8);
1035 if ((uintptr_t)p & 8)
1036 *p = 0;
1037}
1038EXPORT_SYMBOL(cxgb4_inline_tx_skb);
1039
1040static void *inline_tx_skb_header(const struct sk_buff *skb,
1041 const struct sge_txq *q, void *pos,
1042 int length)
1043{
1044 u64 *p;
1045 int left = (void *)q->stat - pos;
1046
1047 if (likely(length <= left)) {
1048 memcpy(pos, skb->data, length);
1049 pos += length;
1050 } else {
1051 memcpy(pos, skb->data, left);
1052 memcpy(q->desc, skb->data + left, length - left);
1053 pos = (void *)q->desc + (length - left);
1054 }
1055
1056 p = PTR_ALIGN(pos, 8);
1057 if ((uintptr_t)p & 8) {
1058 *p = 0;
1059 return p + 1;
1060 }
1061 return p;
1062}
1063
1064
1065
1066
1067
1068static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1069{
1070 int csum_type;
1071 bool inner_hdr_csum = false;
1072 u16 proto, ver;
1073
1074 if (skb->encapsulation &&
1075 (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5))
1076 inner_hdr_csum = true;
1077
1078 if (inner_hdr_csum) {
1079 ver = inner_ip_hdr(skb)->version;
1080 proto = (ver == 4) ? inner_ip_hdr(skb)->protocol :
1081 inner_ipv6_hdr(skb)->nexthdr;
1082 } else {
1083 ver = ip_hdr(skb)->version;
1084 proto = (ver == 4) ? ip_hdr(skb)->protocol :
1085 ipv6_hdr(skb)->nexthdr;
1086 }
1087
1088 if (ver == 4) {
1089 if (proto == IPPROTO_TCP)
1090 csum_type = TX_CSUM_TCPIP;
1091 else if (proto == IPPROTO_UDP)
1092 csum_type = TX_CSUM_UDPIP;
1093 else {
1094nocsum:
1095
1096
1097
1098 return TXPKT_L4CSUM_DIS_F;
1099 }
1100 } else {
1101
1102
1103
1104 if (proto == IPPROTO_TCP)
1105 csum_type = TX_CSUM_TCPIP6;
1106 else if (proto == IPPROTO_UDP)
1107 csum_type = TX_CSUM_UDPIP6;
1108 else
1109 goto nocsum;
1110 }
1111
1112 if (likely(csum_type >= TX_CSUM_TCPIP)) {
1113 int eth_hdr_len, l4_len;
1114 u64 hdr_len;
1115
1116 if (inner_hdr_csum) {
1117
1118
1119
1120 l4_len = skb_inner_network_header_len(skb);
1121 eth_hdr_len = skb_inner_network_offset(skb) - ETH_HLEN;
1122 } else {
1123 l4_len = skb_network_header_len(skb);
1124 eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
1125 }
1126 hdr_len = TXPKT_IPHDR_LEN_V(l4_len);
1127
1128 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
1129 hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1130 else
1131 hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1132 return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
1133 } else {
1134 int start = skb_transport_offset(skb);
1135
1136 return TXPKT_CSUM_TYPE_V(csum_type) |
1137 TXPKT_CSUM_START_V(start) |
1138 TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1139 }
1140}
1141
1142static void eth_txq_stop(struct sge_eth_txq *q)
1143{
1144 netif_tx_stop_queue(q->txq);
1145 q->q.stops++;
1146}
1147
1148static inline void txq_advance(struct sge_txq *q, unsigned int n)
1149{
1150 q->in_use += n;
1151 q->pidx += n;
1152 if (q->pidx >= q->size)
1153 q->pidx -= q->size;
1154}
1155
1156
1157
1158
1159enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb)
1160{
1161 u8 l4_hdr = 0;
1162 enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1163 struct port_info *pi = netdev_priv(skb->dev);
1164 struct adapter *adapter = pi->adapter;
1165
1166 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1167 skb->inner_protocol != htons(ETH_P_TEB))
1168 return tnl_type;
1169
1170 switch (vlan_get_protocol(skb)) {
1171 case htons(ETH_P_IP):
1172 l4_hdr = ip_hdr(skb)->protocol;
1173 break;
1174 case htons(ETH_P_IPV6):
1175 l4_hdr = ipv6_hdr(skb)->nexthdr;
1176 break;
1177 default:
1178 return tnl_type;
1179 }
1180
1181 switch (l4_hdr) {
1182 case IPPROTO_UDP:
1183 if (adapter->vxlan_port == udp_hdr(skb)->dest)
1184 tnl_type = TX_TNL_TYPE_VXLAN;
1185 else if (adapter->geneve_port == udp_hdr(skb)->dest)
1186 tnl_type = TX_TNL_TYPE_GENEVE;
1187 break;
1188 default:
1189 return tnl_type;
1190 }
1191
1192 return tnl_type;
1193}
1194
1195static inline void t6_fill_tnl_lso(struct sk_buff *skb,
1196 struct cpl_tx_tnl_lso *tnl_lso,
1197 enum cpl_tx_tnl_lso_type tnl_type)
1198{
1199 u32 val;
1200 int in_eth_xtra_len;
1201 int l3hdr_len = skb_network_header_len(skb);
1202 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1203 const struct skb_shared_info *ssi = skb_shinfo(skb);
1204 bool v6 = (ip_hdr(skb)->version == 6);
1205
1206 val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) |
1207 CPL_TX_TNL_LSO_FIRST_F |
1208 CPL_TX_TNL_LSO_LAST_F |
1209 (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) |
1210 CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) |
1211 CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) |
1212 (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) |
1213 CPL_TX_TNL_LSO_IPLENSETOUT_F |
1214 (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F);
1215 tnl_lso->op_to_IpIdSplitOut = htonl(val);
1216
1217 tnl_lso->IpIdOffsetOut = 0;
1218
1219
1220 val = skb_inner_mac_header(skb) - skb_mac_header(skb);
1221 in_eth_xtra_len = skb_inner_network_header(skb) -
1222 skb_inner_mac_header(skb) - ETH_HLEN;
1223
1224 switch (tnl_type) {
1225 case TX_TNL_TYPE_VXLAN:
1226 case TX_TNL_TYPE_GENEVE:
1227 tnl_lso->UdpLenSetOut_to_TnlHdrLen =
1228 htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F |
1229 CPL_TX_TNL_LSO_UDPLENSETOUT_F);
1230 break;
1231 default:
1232 tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0;
1233 break;
1234 }
1235
1236 tnl_lso->UdpLenSetOut_to_TnlHdrLen |=
1237 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) |
1238 CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type));
1239
1240 tnl_lso->r1 = 0;
1241
1242 val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) |
1243 CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) |
1244 CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) |
1245 CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4);
1246 tnl_lso->Flow_to_TcpHdrLen = htonl(val);
1247
1248 tnl_lso->IpIdOffset = htons(0);
1249
1250 tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size));
1251 tnl_lso->TCPSeqOffset = htonl(0);
1252 tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len));
1253}
1254
1255
1256
1257
1258
1259
1260
1261
1262static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1263{
1264 u32 wr_mid, ctrl0, op;
1265 u64 cntrl = 0;
1266 u64 *end, *sgl;
1267 int qidx, credits;
1268 unsigned int flits, ndesc;
1269 struct adapter *adap;
1270 struct sge_eth_txq *q;
1271 const struct port_info *pi;
1272 struct fw_eth_tx_pkt_wr *wr;
1273 struct cpl_tx_pkt_core *cpl;
1274 const struct skb_shared_info *ssi;
1275 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1276 bool immediate = false;
1277 int len, max_pkt_len;
1278 bool ptp_enabled = is_ptp_enabled(skb, dev);
1279 unsigned int chip_ver;
1280 enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1281
1282
1283
1284
1285
1286
1287 if (unlikely(skb->len < ETH_HLEN)) {
1288out_free: dev_kfree_skb_any(skb);
1289 return NETDEV_TX_OK;
1290 }
1291
1292
1293 max_pkt_len = ETH_HLEN + dev->mtu;
1294 if (skb_vlan_tagged(skb))
1295 max_pkt_len += VLAN_HLEN;
1296 if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1297 goto out_free;
1298
1299 pi = netdev_priv(dev);
1300 adap = pi->adapter;
1301 ssi = skb_shinfo(skb);
1302#ifdef CONFIG_CHELSIO_IPSEC_INLINE
1303 if (xfrm_offload(skb) && !ssi->gso_size)
1304 return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev);
1305#endif
1306
1307 qidx = skb_get_queue_mapping(skb);
1308 if (ptp_enabled) {
1309 spin_lock(&adap->ptp_lock);
1310 if (!(adap->ptp_tx_skb)) {
1311 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1312 adap->ptp_tx_skb = skb_get(skb);
1313 } else {
1314 spin_unlock(&adap->ptp_lock);
1315 goto out_free;
1316 }
1317 q = &adap->sge.ptptxq;
1318 } else {
1319 q = &adap->sge.ethtxq[qidx + pi->first_qset];
1320 }
1321 skb_tx_timestamp(skb);
1322
1323 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1324
1325 chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
1326 flits = calc_tx_flits(skb, chip_ver);
1327 ndesc = flits_to_desc(flits);
1328 credits = txq_avail(&q->q) - ndesc;
1329
1330 if (unlikely(credits < 0)) {
1331 eth_txq_stop(q);
1332 dev_err(adap->pdev_dev,
1333 "%s: Tx ring %u full while queue awake!\n",
1334 dev->name, qidx);
1335 if (ptp_enabled)
1336 spin_unlock(&adap->ptp_lock);
1337 return NETDEV_TX_BUSY;
1338 }
1339
1340 if (is_eth_imm(skb, chip_ver))
1341 immediate = true;
1342
1343 if (skb->encapsulation && chip_ver > CHELSIO_T5)
1344 tnl_type = cxgb_encap_offload_supported(skb);
1345
1346 if (!immediate &&
1347 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
1348 q->mapping_err++;
1349 if (ptp_enabled)
1350 spin_unlock(&adap->ptp_lock);
1351 goto out_free;
1352 }
1353
1354 wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1355 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1356 eth_txq_stop(q);
1357 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1358 }
1359
1360 wr = (void *)&q->q.desc[q->q.pidx];
1361 wr->equiq_to_len16 = htonl(wr_mid);
1362 wr->r3 = cpu_to_be64(0);
1363 end = (u64 *)wr + flits;
1364
1365 len = immediate ? skb->len : 0;
1366 len += sizeof(*cpl);
1367 if (ssi->gso_size) {
1368 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1369 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1370 int l3hdr_len = skb_network_header_len(skb);
1371 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1372 struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1);
1373
1374 if (tnl_type)
1375 len += sizeof(*tnl_lso);
1376 else
1377 len += sizeof(*lso);
1378
1379 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1380 FW_WR_IMMDLEN_V(len));
1381 if (tnl_type) {
1382 struct iphdr *iph = ip_hdr(skb);
1383
1384 t6_fill_tnl_lso(skb, tnl_lso, tnl_type);
1385 cpl = (void *)(tnl_lso + 1);
1386
1387
1388
1389 if (iph->version == 4) {
1390 iph->check = 0;
1391 iph->tot_len = 0;
1392 iph->check = (u16)(~ip_fast_csum((u8 *)iph,
1393 iph->ihl));
1394 }
1395 if (skb->ip_summed == CHECKSUM_PARTIAL)
1396 cntrl = hwcsum(adap->params.chip, skb);
1397 } else {
1398 lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1399 LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
1400 LSO_IPV6_V(v6) |
1401 LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1402 LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1403 LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1404 lso->ipid_ofst = htons(0);
1405 lso->mss = htons(ssi->gso_size);
1406 lso->seqno_offset = htonl(0);
1407 if (is_t4(adap->params.chip))
1408 lso->len = htonl(skb->len);
1409 else
1410 lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
1411 cpl = (void *)(lso + 1);
1412
1413 if (CHELSIO_CHIP_VERSION(adap->params.chip)
1414 <= CHELSIO_T5)
1415 cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1416 else
1417 cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1418
1419 cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1420 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1421 TXPKT_IPHDR_LEN_V(l3hdr_len);
1422 }
1423 sgl = (u64 *)(cpl + 1);
1424 if (unlikely((u8 *)sgl >= (u8 *)q->q.stat)) {
1425
1426
1427
1428
1429 if (sgl == (u64 *)q->q.stat) {
1430 int left = (u8 *)end - (u8 *)q->q.stat;
1431
1432 end = (void *)q->q.desc + left;
1433 sgl = (void *)q->q.desc;
1434 }
1435 }
1436 q->tso++;
1437 q->tx_cso += ssi->gso_segs;
1438 } else {
1439 if (ptp_enabled)
1440 op = FW_PTP_TX_PKT_WR;
1441 else
1442 op = FW_ETH_TX_PKT_WR;
1443 wr->op_immdlen = htonl(FW_WR_OP_V(op) |
1444 FW_WR_IMMDLEN_V(len));
1445 cpl = (void *)(wr + 1);
1446 sgl = (u64 *)(cpl + 1);
1447 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1448 cntrl = hwcsum(adap->params.chip, skb) |
1449 TXPKT_IPCSUM_DIS_F;
1450 q->tx_cso++;
1451 } else
1452 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1453 }
1454
1455 if (skb_vlan_tag_present(skb)) {
1456 q->vlan_ins++;
1457 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1458 }
1459
1460 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
1461 TXPKT_PF_V(adap->pf);
1462 if (ptp_enabled)
1463 ctrl0 |= TXPKT_TSTAMP_F;
1464#ifdef CONFIG_CHELSIO_T4_DCB
1465 if (is_t4(adap->params.chip))
1466 ctrl0 |= TXPKT_OVLAN_IDX_V(q->dcb_prio);
1467 else
1468 ctrl0 |= TXPKT_T5_OVLAN_IDX_V(q->dcb_prio);
1469#endif
1470 cpl->ctrl0 = htonl(ctrl0);
1471 cpl->pack = htons(0);
1472 cpl->len = htons(skb->len);
1473 cpl->ctrl1 = cpu_to_be64(cntrl);
1474
1475 if (immediate) {
1476 cxgb4_inline_tx_skb(skb, &q->q, sgl);
1477 dev_consume_skb_any(skb);
1478 } else {
1479 int last_desc;
1480
1481 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, 0, addr);
1482 skb_orphan(skb);
1483
1484 last_desc = q->q.pidx + ndesc - 1;
1485 if (last_desc >= q->q.size)
1486 last_desc -= q->q.size;
1487 q->q.sdesc[last_desc].skb = skb;
1488 q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
1489 }
1490
1491 txq_advance(&q->q, ndesc);
1492
1493 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1494 if (ptp_enabled)
1495 spin_unlock(&adap->ptp_lock);
1496 return NETDEV_TX_OK;
1497}
1498
1499
1500enum {
1501
1502
1503
1504
1505
1506
1507 EQ_UNIT = SGE_EQ_IDXSIZE,
1508 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1509 TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1510
1511 T4VF_ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1512 sizeof(struct cpl_tx_pkt_lso_core) +
1513 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
1514};
1515
1516
1517
1518
1519
1520
1521
1522
1523static inline int t4vf_is_eth_imm(const struct sk_buff *skb)
1524{
1525
1526
1527
1528
1529
1530
1531 return false;
1532}
1533
1534
1535
1536
1537
1538
1539
1540
1541static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff *skb)
1542{
1543 unsigned int flits;
1544
1545
1546
1547
1548
1549 if (t4vf_is_eth_imm(skb))
1550 return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
1551 sizeof(__be64));
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
1562 if (skb_shinfo(skb)->gso_size)
1563 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1564 sizeof(struct cpl_tx_pkt_lso_core) +
1565 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1566 else
1567 flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1568 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1569 return flits;
1570}
1571
1572
1573
1574
1575
1576
1577
1578
1579static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb,
1580 struct net_device *dev)
1581{
1582 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1583 const struct skb_shared_info *ssi;
1584 struct fw_eth_tx_pkt_vm_wr *wr;
1585 int qidx, credits, max_pkt_len;
1586 struct cpl_tx_pkt_core *cpl;
1587 const struct port_info *pi;
1588 unsigned int flits, ndesc;
1589 struct sge_eth_txq *txq;
1590 struct adapter *adapter;
1591 u64 cntrl, *end;
1592 u32 wr_mid;
1593 const size_t fw_hdr_copy_len = sizeof(wr->ethmacdst) +
1594 sizeof(wr->ethmacsrc) +
1595 sizeof(wr->ethtype) +
1596 sizeof(wr->vlantci);
1597
1598
1599
1600
1601
1602
1603 if (unlikely(skb->len < fw_hdr_copy_len))
1604 goto out_free;
1605
1606
1607 max_pkt_len = ETH_HLEN + dev->mtu;
1608 if (skb_vlan_tag_present(skb))
1609 max_pkt_len += VLAN_HLEN;
1610 if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1611 goto out_free;
1612
1613
1614 pi = netdev_priv(dev);
1615 adapter = pi->adapter;
1616 qidx = skb_get_queue_mapping(skb);
1617 WARN_ON(qidx >= pi->nqsets);
1618 txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1619
1620
1621
1622
1623 cxgb4_reclaim_completed_tx(adapter, &txq->q, true);
1624
1625
1626
1627
1628
1629 flits = t4vf_calc_tx_flits(skb);
1630 ndesc = flits_to_desc(flits);
1631 credits = txq_avail(&txq->q) - ndesc;
1632
1633 if (unlikely(credits < 0)) {
1634
1635
1636
1637
1638
1639 eth_txq_stop(txq);
1640 dev_err(adapter->pdev_dev,
1641 "%s: TX ring %u full while queue awake!\n",
1642 dev->name, qidx);
1643 return NETDEV_TX_BUSY;
1644 }
1645
1646 if (!t4vf_is_eth_imm(skb) &&
1647 unlikely(cxgb4_map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1648
1649
1650
1651
1652 txq->mapping_err++;
1653 goto out_free;
1654 }
1655
1656 wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1657 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1658
1659
1660
1661
1662
1663
1664
1665
1666 eth_txq_stop(txq);
1667 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1668 }
1669
1670
1671
1672
1673
1674
1675 WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1676 wr = (void *)&txq->q.desc[txq->q.pidx];
1677 wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1678 wr->r3[0] = cpu_to_be32(0);
1679 wr->r3[1] = cpu_to_be32(0);
1680 skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1681 end = (u64 *)wr + flits;
1682
1683
1684
1685
1686
1687 ssi = skb_shinfo(skb);
1688 if (ssi->gso_size) {
1689 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1690 bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1691 int l3hdr_len = skb_network_header_len(skb);
1692 int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1693
1694 wr->op_immdlen =
1695 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1696 FW_WR_IMMDLEN_V(sizeof(*lso) +
1697 sizeof(*cpl)));
1698
1699 lso->lso_ctrl =
1700 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1701 LSO_FIRST_SLICE_F |
1702 LSO_LAST_SLICE_F |
1703 LSO_IPV6_V(v6) |
1704 LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1705 LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1706 LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1707 lso->ipid_ofst = cpu_to_be16(0);
1708 lso->mss = cpu_to_be16(ssi->gso_size);
1709 lso->seqno_offset = cpu_to_be32(0);
1710 if (is_t4(adapter->params.chip))
1711 lso->len = cpu_to_be32(skb->len);
1712 else
1713 lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len));
1714
1715
1716
1717
1718 cpl = (void *)(lso + 1);
1719
1720 if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
1721 cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1722 else
1723 cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1724
1725 cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1726 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1727 TXPKT_IPHDR_LEN_V(l3hdr_len);
1728 txq->tso++;
1729 txq->tx_cso += ssi->gso_segs;
1730 } else {
1731 int len;
1732
1733 len = (t4vf_is_eth_imm(skb)
1734 ? skb->len + sizeof(*cpl)
1735 : sizeof(*cpl));
1736 wr->op_immdlen =
1737 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1738 FW_WR_IMMDLEN_V(len));
1739
1740
1741
1742
1743 cpl = (void *)(wr + 1);
1744 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1745 cntrl = hwcsum(adapter->params.chip, skb) |
1746 TXPKT_IPCSUM_DIS_F;
1747 txq->tx_cso++;
1748 } else {
1749 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1750 }
1751 }
1752
1753
1754
1755
1756 if (skb_vlan_tag_present(skb)) {
1757 txq->vlan_ins++;
1758 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1759 }
1760
1761
1762 cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) |
1763 TXPKT_INTF_V(pi->port_id) |
1764 TXPKT_PF_V(0));
1765 cpl->pack = cpu_to_be16(0);
1766 cpl->len = cpu_to_be16(skb->len);
1767 cpl->ctrl1 = cpu_to_be64(cntrl);
1768
1769
1770
1771
1772 if (t4vf_is_eth_imm(skb)) {
1773
1774
1775
1776 cxgb4_inline_tx_skb(skb, &txq->q, cpl + 1);
1777 dev_consume_skb_any(skb);
1778 } else {
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815 struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1816 struct sge_txq *tq = &txq->q;
1817 int last_desc;
1818
1819
1820
1821
1822
1823
1824
1825 if (unlikely((void *)sgl == (void *)tq->stat)) {
1826 sgl = (void *)tq->desc;
1827 end = (void *)((void *)tq->desc +
1828 ((void *)end - (void *)tq->stat));
1829 }
1830
1831 cxgb4_write_sgl(skb, tq, sgl, end, 0, addr);
1832 skb_orphan(skb);
1833
1834 last_desc = tq->pidx + ndesc - 1;
1835 if (last_desc >= tq->size)
1836 last_desc -= tq->size;
1837 tq->sdesc[last_desc].skb = skb;
1838 tq->sdesc[last_desc].sgl = sgl;
1839 }
1840
1841
1842
1843
1844 txq_advance(&txq->q, ndesc);
1845
1846 cxgb4_ring_tx_db(adapter, &txq->q, ndesc);
1847 return NETDEV_TX_OK;
1848
1849out_free:
1850
1851
1852
1853 dev_kfree_skb_any(skb);
1854 return NETDEV_TX_OK;
1855}
1856
1857netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev)
1858{
1859 struct port_info *pi = netdev_priv(dev);
1860
1861 if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM))
1862 return cxgb4_vf_eth_xmit(skb, dev);
1863
1864 return cxgb4_eth_xmit(skb, dev);
1865}
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1876{
1877 int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
1878 int reclaim = hw_cidx - q->cidx;
1879
1880 if (reclaim < 0)
1881 reclaim += q->size;
1882
1883 q->in_use -= reclaim;
1884 q->cidx = hw_cidx;
1885}
1886
1887
1888
1889
1890
1891
1892
1893static inline int is_imm(const struct sk_buff *skb)
1894{
1895 return skb->len <= MAX_CTRL_WR_LEN;
1896}
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr)
1909{
1910 reclaim_completed_tx_imm(&q->q);
1911 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
1912 wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
1913 q->q.stops++;
1914 q->full = 1;
1915 }
1916}
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb)
1927{
1928 unsigned int ndesc;
1929 struct fw_wr_hdr *wr;
1930
1931 if (unlikely(!is_imm(skb))) {
1932 WARN_ON(1);
1933 dev_kfree_skb(skb);
1934 return NET_XMIT_DROP;
1935 }
1936
1937 ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc));
1938 spin_lock(&q->sendq.lock);
1939
1940 if (unlikely(q->full)) {
1941 skb->priority = ndesc;
1942 __skb_queue_tail(&q->sendq, skb);
1943 spin_unlock(&q->sendq.lock);
1944 return NET_XMIT_CN;
1945 }
1946
1947 wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
1948 cxgb4_inline_tx_skb(skb, &q->q, wr);
1949
1950 txq_advance(&q->q, ndesc);
1951 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES))
1952 ctrlq_check_stop(q, wr);
1953
1954 cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
1955 spin_unlock(&q->sendq.lock);
1956
1957 kfree_skb(skb);
1958 return NET_XMIT_SUCCESS;
1959}
1960
1961
1962
1963
1964
1965
1966
1967static void restart_ctrlq(unsigned long data)
1968{
1969 struct sk_buff *skb;
1970 unsigned int written = 0;
1971 struct sge_ctrl_txq *q = (struct sge_ctrl_txq *)data;
1972
1973 spin_lock(&q->sendq.lock);
1974 reclaim_completed_tx_imm(&q->q);
1975 BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES);
1976
1977 while ((skb = __skb_dequeue(&q->sendq)) != NULL) {
1978 struct fw_wr_hdr *wr;
1979 unsigned int ndesc = skb->priority;
1980
1981 written += ndesc;
1982
1983
1984
1985 wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
1986 txq_advance(&q->q, ndesc);
1987 spin_unlock(&q->sendq.lock);
1988
1989 cxgb4_inline_tx_skb(skb, &q->q, wr);
1990 kfree_skb(skb);
1991
1992 if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
1993 unsigned long old = q->q.stops;
1994
1995 ctrlq_check_stop(q, wr);
1996 if (q->q.stops != old) {
1997 spin_lock(&q->sendq.lock);
1998 goto ringdb;
1999 }
2000 }
2001 if (written > 16) {
2002 cxgb4_ring_tx_db(q->adap, &q->q, written);
2003 written = 0;
2004 }
2005 spin_lock(&q->sendq.lock);
2006 }
2007 q->full = 0;
2008ringdb:
2009 if (written)
2010 cxgb4_ring_tx_db(q->adap, &q->q, written);
2011 spin_unlock(&q->sendq.lock);
2012}
2013
2014
2015
2016
2017
2018
2019
2020
2021int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
2022{
2023 int ret;
2024
2025 local_bh_disable();
2026 ret = ctrl_xmit(&adap->sge.ctrlq[0], skb);
2027 local_bh_enable();
2028 return ret;
2029}
2030
2031
2032
2033
2034
2035
2036
2037
2038static inline int is_ofld_imm(const struct sk_buff *skb)
2039{
2040 struct work_request_hdr *req = (struct work_request_hdr *)skb->data;
2041 unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi));
2042
2043 if (opcode == FW_CRYPTO_LOOKASIDE_WR)
2044 return skb->len <= SGE_MAX_WR_LEN;
2045 else
2046 return skb->len <= MAX_IMM_TX_PKT_LEN;
2047}
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
2058{
2059 unsigned int flits, cnt;
2060
2061 if (is_ofld_imm(skb))
2062 return DIV_ROUND_UP(skb->len, 8);
2063
2064 flits = skb_transport_offset(skb) / 8U;
2065 cnt = skb_shinfo(skb)->nr_frags;
2066 if (skb_tail_pointer(skb) != skb_transport_header(skb))
2067 cnt++;
2068 return flits + sgl_len(cnt);
2069}
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080static void txq_stop_maperr(struct sge_uld_txq *q)
2081{
2082 q->mapping_err++;
2083 q->q.stops++;
2084 set_bit(q->q.cntxt_id - q->adap->sge.egr_start,
2085 q->adap->sge.txq_maperr);
2086}
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096static void ofldtxq_stop(struct sge_uld_txq *q, struct fw_wr_hdr *wr)
2097{
2098 wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
2099 q->q.stops++;
2100 q->full = 1;
2101}
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121static void service_ofldq(struct sge_uld_txq *q)
2122{
2123 u64 *pos, *before, *end;
2124 int credits;
2125 struct sk_buff *skb;
2126 struct sge_txq *txq;
2127 unsigned int left;
2128 unsigned int written = 0;
2129 unsigned int flits, ndesc;
2130
2131
2132
2133
2134
2135
2136
2137 if (q->service_ofldq_running)
2138 return;
2139 q->service_ofldq_running = true;
2140
2141 while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) {
2142
2143
2144
2145
2146
2147
2148
2149 spin_unlock(&q->sendq.lock);
2150
2151 cxgb4_reclaim_completed_tx(q->adap, &q->q, false);
2152
2153 flits = skb->priority;
2154 ndesc = flits_to_desc(flits);
2155 credits = txq_avail(&q->q) - ndesc;
2156 BUG_ON(credits < 0);
2157 if (unlikely(credits < TXQ_STOP_THRES))
2158 ofldtxq_stop(q, (struct fw_wr_hdr *)skb->data);
2159
2160 pos = (u64 *)&q->q.desc[q->q.pidx];
2161 if (is_ofld_imm(skb))
2162 cxgb4_inline_tx_skb(skb, &q->q, pos);
2163 else if (cxgb4_map_skb(q->adap->pdev_dev, skb,
2164 (dma_addr_t *)skb->head)) {
2165 txq_stop_maperr(q);
2166 spin_lock(&q->sendq.lock);
2167 break;
2168 } else {
2169 int last_desc, hdr_len = skb_transport_offset(skb);
2170
2171
2172
2173
2174 before = (u64 *)pos;
2175 end = (u64 *)pos + flits;
2176 txq = &q->q;
2177 pos = (void *)inline_tx_skb_header(skb, &q->q,
2178 (void *)pos,
2179 hdr_len);
2180 if (before > (u64 *)pos) {
2181 left = (u8 *)end - (u8 *)txq->stat;
2182 end = (void *)txq->desc + left;
2183 }
2184
2185
2186
2187
2188
2189 if (pos == (u64 *)txq->stat) {
2190 left = (u8 *)end - (u8 *)txq->stat;
2191 end = (void *)txq->desc + left;
2192 pos = (void *)txq->desc;
2193 }
2194
2195 cxgb4_write_sgl(skb, &q->q, (void *)pos,
2196 end, hdr_len,
2197 (dma_addr_t *)skb->head);
2198#ifdef CONFIG_NEED_DMA_MAP_STATE
2199 skb->dev = q->adap->port[0];
2200 skb->destructor = deferred_unmap_destructor;
2201#endif
2202 last_desc = q->q.pidx + ndesc - 1;
2203 if (last_desc >= q->q.size)
2204 last_desc -= q->q.size;
2205 q->q.sdesc[last_desc].skb = skb;
2206 }
2207
2208 txq_advance(&q->q, ndesc);
2209 written += ndesc;
2210 if (unlikely(written > 32)) {
2211 cxgb4_ring_tx_db(q->adap, &q->q, written);
2212 written = 0;
2213 }
2214
2215
2216
2217
2218
2219
2220 spin_lock(&q->sendq.lock);
2221 __skb_unlink(skb, &q->sendq);
2222 if (is_ofld_imm(skb))
2223 kfree_skb(skb);
2224 }
2225 if (likely(written))
2226 cxgb4_ring_tx_db(q->adap, &q->q, written);
2227
2228
2229
2230
2231 q->service_ofldq_running = false;
2232}
2233
2234
2235
2236
2237
2238
2239
2240
2241static int ofld_xmit(struct sge_uld_txq *q, struct sk_buff *skb)
2242{
2243 skb->priority = calc_tx_flits_ofld(skb);
2244 spin_lock(&q->sendq.lock);
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254 __skb_queue_tail(&q->sendq, skb);
2255 if (q->sendq.qlen == 1)
2256 service_ofldq(q);
2257
2258 spin_unlock(&q->sendq.lock);
2259 return NET_XMIT_SUCCESS;
2260}
2261
2262
2263
2264
2265
2266
2267
2268static void restart_ofldq(unsigned long data)
2269{
2270 struct sge_uld_txq *q = (struct sge_uld_txq *)data;
2271
2272 spin_lock(&q->sendq.lock);
2273 q->full = 0;
2274 service_ofldq(q);
2275 spin_unlock(&q->sendq.lock);
2276}
2277
2278
2279
2280
2281
2282
2283
2284
2285static inline unsigned int skb_txq(const struct sk_buff *skb)
2286{
2287 return skb->queue_mapping >> 1;
2288}
2289
2290
2291
2292
2293
2294
2295
2296
2297static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb)
2298{
2299 return skb->queue_mapping & 1;
2300}
2301
2302static inline int uld_send(struct adapter *adap, struct sk_buff *skb,
2303 unsigned int tx_uld_type)
2304{
2305 struct sge_uld_txq_info *txq_info;
2306 struct sge_uld_txq *txq;
2307 unsigned int idx = skb_txq(skb);
2308
2309 if (unlikely(is_ctrl_pkt(skb))) {
2310
2311 if (adap->tids.nsftids)
2312 idx = 0;
2313 return ctrl_xmit(&adap->sge.ctrlq[idx], skb);
2314 }
2315
2316 txq_info = adap->sge.uld_txq_info[tx_uld_type];
2317 if (unlikely(!txq_info)) {
2318 WARN_ON(true);
2319 return NET_XMIT_DROP;
2320 }
2321
2322 txq = &txq_info->uldtxq[idx];
2323 return ofld_xmit(txq, skb);
2324}
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335int t4_ofld_send(struct adapter *adap, struct sk_buff *skb)
2336{
2337 int ret;
2338
2339 local_bh_disable();
2340 ret = uld_send(adap, skb, CXGB4_TX_OFLD);
2341 local_bh_enable();
2342 return ret;
2343}
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb)
2354{
2355 return t4_ofld_send(netdev2adap(dev), skb);
2356}
2357EXPORT_SYMBOL(cxgb4_ofld_send);
2358
2359static void *inline_tx_header(const void *src,
2360 const struct sge_txq *q,
2361 void *pos, int length)
2362{
2363 int left = (void *)q->stat - pos;
2364 u64 *p;
2365
2366 if (likely(length <= left)) {
2367 memcpy(pos, src, length);
2368 pos += length;
2369 } else {
2370 memcpy(pos, src, left);
2371 memcpy(q->desc, src + left, length - left);
2372 pos = (void *)q->desc + (length - left);
2373 }
2374
2375 p = PTR_ALIGN(pos, 8);
2376 if ((uintptr_t)p & 8) {
2377 *p = 0;
2378 return p + 1;
2379 }
2380 return p;
2381}
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391static int ofld_xmit_direct(struct sge_uld_txq *q, const void *src,
2392 unsigned int len)
2393{
2394 unsigned int ndesc;
2395 int credits;
2396 u64 *pos;
2397
2398
2399 if (len > MAX_IMM_OFLD_TX_DATA_WR_LEN) {
2400 WARN_ON(1);
2401 return NET_XMIT_DROP;
2402 }
2403
2404
2405
2406
2407
2408 if (!spin_trylock(&q->sendq.lock))
2409 return NET_XMIT_DROP;
2410
2411 if (q->full || !skb_queue_empty(&q->sendq) ||
2412 q->service_ofldq_running) {
2413 spin_unlock(&q->sendq.lock);
2414 return NET_XMIT_DROP;
2415 }
2416 ndesc = flits_to_desc(DIV_ROUND_UP(len, 8));
2417 credits = txq_avail(&q->q) - ndesc;
2418 pos = (u64 *)&q->q.desc[q->q.pidx];
2419
2420
2421 inline_tx_header(src, &q->q, pos, len);
2422 if (unlikely(credits < TXQ_STOP_THRES))
2423 ofldtxq_stop(q, (struct fw_wr_hdr *)pos);
2424 txq_advance(&q->q, ndesc);
2425 cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
2426
2427 spin_unlock(&q->sendq.lock);
2428 return NET_XMIT_SUCCESS;
2429}
2430
2431int cxgb4_immdata_send(struct net_device *dev, unsigned int idx,
2432 const void *src, unsigned int len)
2433{
2434 struct sge_uld_txq_info *txq_info;
2435 struct sge_uld_txq *txq;
2436 struct adapter *adap;
2437 int ret;
2438
2439 adap = netdev2adap(dev);
2440
2441 local_bh_disable();
2442 txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
2443 if (unlikely(!txq_info)) {
2444 WARN_ON(true);
2445 local_bh_enable();
2446 return NET_XMIT_DROP;
2447 }
2448 txq = &txq_info->uldtxq[idx];
2449
2450 ret = ofld_xmit_direct(txq, src, len);
2451 local_bh_enable();
2452 return net_xmit_eval(ret);
2453}
2454EXPORT_SYMBOL(cxgb4_immdata_send);
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465static int t4_crypto_send(struct adapter *adap, struct sk_buff *skb)
2466{
2467 int ret;
2468
2469 local_bh_disable();
2470 ret = uld_send(adap, skb, CXGB4_TX_CRYPTO);
2471 local_bh_enable();
2472 return ret;
2473}
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb)
2484{
2485 return t4_crypto_send(netdev2adap(dev), skb);
2486}
2487EXPORT_SYMBOL(cxgb4_crypto_send);
2488
2489static inline void copy_frags(struct sk_buff *skb,
2490 const struct pkt_gl *gl, unsigned int offset)
2491{
2492 int i;
2493
2494
2495 __skb_fill_page_desc(skb, 0, gl->frags[0].page,
2496 gl->frags[0].offset + offset,
2497 gl->frags[0].size - offset);
2498 skb_shinfo(skb)->nr_frags = gl->nfrags;
2499 for (i = 1; i < gl->nfrags; i++)
2500 __skb_fill_page_desc(skb, i, gl->frags[i].page,
2501 gl->frags[i].offset,
2502 gl->frags[i].size);
2503
2504
2505 get_page(gl->frags[gl->nfrags - 1].page);
2506}
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl,
2518 unsigned int skb_len, unsigned int pull_len)
2519{
2520 struct sk_buff *skb;
2521
2522
2523
2524
2525
2526
2527 if (gl->tot_len <= RX_COPY_THRES) {
2528 skb = dev_alloc_skb(gl->tot_len);
2529 if (unlikely(!skb))
2530 goto out;
2531 __skb_put(skb, gl->tot_len);
2532 skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
2533 } else {
2534 skb = dev_alloc_skb(skb_len);
2535 if (unlikely(!skb))
2536 goto out;
2537 __skb_put(skb, pull_len);
2538 skb_copy_to_linear_data(skb, gl->va, pull_len);
2539
2540 copy_frags(skb, gl, pull_len);
2541 skb->len = gl->tot_len;
2542 skb->data_len = skb->len - pull_len;
2543 skb->truesize += skb->data_len;
2544 }
2545out: return skb;
2546}
2547EXPORT_SYMBOL(cxgb4_pktgl_to_skb);
2548
2549
2550
2551
2552
2553
2554
2555
2556static void t4_pktgl_free(const struct pkt_gl *gl)
2557{
2558 int n;
2559 const struct page_frag *p;
2560
2561 for (p = gl->frags, n = gl->nfrags - 1; n--; p++)
2562 put_page(p->page);
2563}
2564
2565
2566
2567
2568
2569static noinline int handle_trace_pkt(struct adapter *adap,
2570 const struct pkt_gl *gl)
2571{
2572 struct sk_buff *skb;
2573
2574 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
2575 if (unlikely(!skb)) {
2576 t4_pktgl_free(gl);
2577 return 0;
2578 }
2579
2580 if (is_t4(adap->params.chip))
2581 __skb_pull(skb, sizeof(struct cpl_trace_pkt));
2582 else
2583 __skb_pull(skb, sizeof(struct cpl_t5_trace_pkt));
2584
2585 skb_reset_mac_header(skb);
2586 skb->protocol = htons(0xffff);
2587 skb->dev = adap->port[0];
2588 netif_receive_skb(skb);
2589 return 0;
2590}
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap,
2602 struct skb_shared_hwtstamps *hwtstamps,
2603 u64 sgetstamp)
2604{
2605 u64 ns;
2606 u64 tmp = (sgetstamp * 1000 * 1000 + adap->params.vpd.cclk / 2);
2607
2608 ns = div_u64(tmp, adap->params.vpd.cclk);
2609
2610 memset(hwtstamps, 0, sizeof(*hwtstamps));
2611 hwtstamps->hwtstamp = ns_to_ktime(ns);
2612}
2613
2614static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
2615 const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len)
2616{
2617 struct adapter *adapter = rxq->rspq.adap;
2618 struct sge *s = &adapter->sge;
2619 struct port_info *pi;
2620 int ret;
2621 struct sk_buff *skb;
2622
2623 skb = napi_get_frags(&rxq->rspq.napi);
2624 if (unlikely(!skb)) {
2625 t4_pktgl_free(gl);
2626 rxq->stats.rx_drops++;
2627 return;
2628 }
2629
2630 copy_frags(skb, gl, s->pktshift);
2631 if (tnl_hdr_len)
2632 skb->csum_level = 1;
2633 skb->len = gl->tot_len - s->pktshift;
2634 skb->data_len = skb->len;
2635 skb->truesize += skb->data_len;
2636 skb->ip_summed = CHECKSUM_UNNECESSARY;
2637 skb_record_rx_queue(skb, rxq->rspq.idx);
2638 skb_mark_napi_id(skb, &rxq->rspq.napi);
2639 pi = netdev_priv(skb->dev);
2640 if (pi->rxtstamp)
2641 cxgb4_sgetim_to_hwtstamp(adapter, skb_hwtstamps(skb),
2642 gl->sgetstamp);
2643 if (rxq->rspq.netdev->features & NETIF_F_RXHASH)
2644 skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
2645 PKT_HASH_TYPE_L3);
2646
2647 if (unlikely(pkt->vlan_ex)) {
2648 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2649 rxq->stats.vlan_ex++;
2650 }
2651 ret = napi_gro_frags(&rxq->rspq.napi);
2652 if (ret == GRO_HELD)
2653 rxq->stats.lro_pkts++;
2654 else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
2655 rxq->stats.lro_merged++;
2656 rxq->stats.pkts++;
2657 rxq->stats.rx_cso++;
2658}
2659
2660enum {
2661 RX_NON_PTP_PKT = 0,
2662 RX_PTP_PKT_SUC = 1,
2663 RX_PTP_PKT_ERR = 2
2664};
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674static noinline int t4_systim_to_hwstamp(struct adapter *adapter,
2675 struct sk_buff *skb)
2676{
2677 struct skb_shared_hwtstamps *hwtstamps;
2678 struct cpl_rx_mps_pkt *cpl = NULL;
2679 unsigned char *data;
2680 int offset;
2681
2682 cpl = (struct cpl_rx_mps_pkt *)skb->data;
2683 if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl->op_to_r1_hi)) &
2684 X_CPL_RX_MPS_PKT_TYPE_PTP))
2685 return RX_PTP_PKT_ERR;
2686
2687 data = skb->data + sizeof(*cpl);
2688 skb_pull(skb, 2 * sizeof(u64) + sizeof(struct cpl_rx_mps_pkt));
2689 offset = ETH_HLEN + IPV4_HLEN(skb->data) + UDP_HLEN;
2690 if (skb->len < offset + OFF_PTP_SEQUENCE_ID + sizeof(short))
2691 return RX_PTP_PKT_ERR;
2692
2693 hwtstamps = skb_hwtstamps(skb);
2694 memset(hwtstamps, 0, sizeof(*hwtstamps));
2695 hwtstamps->hwtstamp = ns_to_ktime(be64_to_cpu(*((u64 *)data)));
2696
2697 return RX_PTP_PKT_SUC;
2698}
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708static int t4_rx_hststamp(struct adapter *adapter, const __be64 *rsp,
2709 struct sge_eth_rxq *rxq, struct sk_buff *skb)
2710{
2711 int ret;
2712
2713 if (unlikely((*(u8 *)rsp == CPL_RX_MPS_PKT) &&
2714 !is_t4(adapter->params.chip))) {
2715 ret = t4_systim_to_hwstamp(adapter, skb);
2716 if (ret == RX_PTP_PKT_ERR) {
2717 kfree_skb(skb);
2718 rxq->stats.rx_drops++;
2719 }
2720 return ret;
2721 }
2722 return RX_NON_PTP_PKT;
2723}
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733static int t4_tx_hststamp(struct adapter *adapter, struct sk_buff *skb,
2734 struct net_device *dev)
2735{
2736 struct port_info *pi = netdev_priv(dev);
2737
2738 if (!is_t4(adapter->params.chip) && adapter->ptp_tx_skb) {
2739 cxgb4_ptp_read_hwstamp(adapter, pi);
2740 kfree_skb(skb);
2741 return 0;
2742 }
2743 return 1;
2744}
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
2755 const struct pkt_gl *si)
2756{
2757 bool csum_ok;
2758 struct sk_buff *skb;
2759 const struct cpl_rx_pkt *pkt;
2760 struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2761 struct adapter *adapter = q->adap;
2762 struct sge *s = &q->adap->sge;
2763 int cpl_trace_pkt = is_t4(q->adap->params.chip) ?
2764 CPL_TRACE_PKT : CPL_TRACE_PKT_T5;
2765 u16 err_vec, tnl_hdr_len = 0;
2766 struct port_info *pi;
2767 int ret = 0;
2768
2769 if (unlikely(*(u8 *)rsp == cpl_trace_pkt))
2770 return handle_trace_pkt(q->adap, si);
2771
2772 pkt = (const struct cpl_rx_pkt *)rsp;
2773
2774 if (q->adap->params.tp.rx_pkt_encap) {
2775 err_vec = T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt->err_vec));
2776 tnl_hdr_len = T6_RX_TNLHDR_LEN_G(ntohs(pkt->err_vec));
2777 } else {
2778 err_vec = be16_to_cpu(pkt->err_vec);
2779 }
2780
2781 csum_ok = pkt->csum_calc && !err_vec &&
2782 (q->netdev->features & NETIF_F_RXCSUM);
2783
2784 if (err_vec)
2785 rxq->stats.bad_rx_pkts++;
2786
2787 if (((pkt->l2info & htonl(RXF_TCP_F)) ||
2788 tnl_hdr_len) &&
2789 (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
2790 do_gro(rxq, si, pkt, tnl_hdr_len);
2791 return 0;
2792 }
2793
2794 skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN);
2795 if (unlikely(!skb)) {
2796 t4_pktgl_free(si);
2797 rxq->stats.rx_drops++;
2798 return 0;
2799 }
2800 pi = netdev_priv(q->netdev);
2801
2802
2803 if (unlikely(pi->ptp_enable)) {
2804 ret = t4_rx_hststamp(adapter, rsp, rxq, skb);
2805 if (ret == RX_PTP_PKT_ERR)
2806 return 0;
2807 }
2808 if (likely(!ret))
2809 __skb_pull(skb, s->pktshift);
2810
2811
2812 if (unlikely(pi->ptp_enable && !ret &&
2813 (pkt->l2info & htonl(RXF_UDP_F)) &&
2814 cxgb4_ptp_is_ptp_rx(skb))) {
2815 if (!t4_tx_hststamp(adapter, skb, q->netdev))
2816 return 0;
2817 }
2818
2819 skb->protocol = eth_type_trans(skb, q->netdev);
2820 skb_record_rx_queue(skb, q->idx);
2821 if (skb->dev->features & NETIF_F_RXHASH)
2822 skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
2823 PKT_HASH_TYPE_L3);
2824
2825 rxq->stats.pkts++;
2826
2827 if (pi->rxtstamp)
2828 cxgb4_sgetim_to_hwtstamp(q->adap, skb_hwtstamps(skb),
2829 si->sgetstamp);
2830 if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) {
2831 if (!pkt->ip_frag) {
2832 skb->ip_summed = CHECKSUM_UNNECESSARY;
2833 rxq->stats.rx_cso++;
2834 } else if (pkt->l2info & htonl(RXF_IP_F)) {
2835 __sum16 c = (__force __sum16)pkt->csum;
2836 skb->csum = csum_unfold(c);
2837
2838 if (tnl_hdr_len) {
2839 skb->ip_summed = CHECKSUM_UNNECESSARY;
2840 skb->csum_level = 1;
2841 } else {
2842 skb->ip_summed = CHECKSUM_COMPLETE;
2843 }
2844 rxq->stats.rx_cso++;
2845 }
2846 } else
2847 skb_checksum_none_assert(skb);
2848
2849 if (unlikely(pkt->vlan_ex)) {
2850 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2851 rxq->stats.vlan_ex++;
2852 }
2853 skb_mark_napi_id(skb, &q->napi);
2854 netif_receive_skb(skb);
2855 return 0;
2856}
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q,
2874 int frags)
2875{
2876 struct rx_sw_desc *d;
2877
2878 while (frags--) {
2879 if (q->cidx == 0)
2880 q->cidx = q->size - 1;
2881 else
2882 q->cidx--;
2883 d = &q->sdesc[q->cidx];
2884 d->page = si->frags[frags].page;
2885 d->dma_addr |= RX_UNMAPPED_BUF;
2886 q->avail++;
2887 }
2888}
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898static inline bool is_new_response(const struct rsp_ctrl *r,
2899 const struct sge_rspq *q)
2900{
2901 return (r->type_gen >> RSPD_GEN_S) == q->gen;
2902}
2903
2904
2905
2906
2907
2908
2909
2910static inline void rspq_next(struct sge_rspq *q)
2911{
2912 q->cur_desc = (void *)q->cur_desc + q->iqe_len;
2913 if (unlikely(++q->cidx == q->size)) {
2914 q->cidx = 0;
2915 q->gen ^= 1;
2916 q->cur_desc = q->desc;
2917 }
2918}
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933static int process_responses(struct sge_rspq *q, int budget)
2934{
2935 int ret, rsp_type;
2936 int budget_left = budget;
2937 const struct rsp_ctrl *rc;
2938 struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2939 struct adapter *adapter = q->adap;
2940 struct sge *s = &adapter->sge;
2941
2942 while (likely(budget_left)) {
2943 rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
2944 if (!is_new_response(rc, q)) {
2945 if (q->flush_handler)
2946 q->flush_handler(q);
2947 break;
2948 }
2949
2950 dma_rmb();
2951 rsp_type = RSPD_TYPE_G(rc->type_gen);
2952 if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
2953 struct page_frag *fp;
2954 struct pkt_gl si;
2955 const struct rx_sw_desc *rsd;
2956 u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags;
2957
2958 if (len & RSPD_NEWBUF_F) {
2959 if (likely(q->offset > 0)) {
2960 free_rx_bufs(q->adap, &rxq->fl, 1);
2961 q->offset = 0;
2962 }
2963 len = RSPD_LEN_G(len);
2964 }
2965 si.tot_len = len;
2966
2967
2968 for (frags = 0, fp = si.frags; ; frags++, fp++) {
2969 rsd = &rxq->fl.sdesc[rxq->fl.cidx];
2970 bufsz = get_buf_size(adapter, rsd);
2971 fp->page = rsd->page;
2972 fp->offset = q->offset;
2973 fp->size = min(bufsz, len);
2974 len -= fp->size;
2975 if (!len)
2976 break;
2977 unmap_rx_buf(q->adap, &rxq->fl);
2978 }
2979
2980 si.sgetstamp = SGE_TIMESTAMP_G(
2981 be64_to_cpu(rc->last_flit));
2982
2983
2984
2985
2986 dma_sync_single_for_cpu(q->adap->pdev_dev,
2987 get_buf_addr(rsd),
2988 fp->size, DMA_FROM_DEVICE);
2989
2990 si.va = page_address(si.frags[0].page) +
2991 si.frags[0].offset;
2992 prefetch(si.va);
2993
2994 si.nfrags = frags + 1;
2995 ret = q->handler(q, q->cur_desc, &si);
2996 if (likely(ret == 0))
2997 q->offset += ALIGN(fp->size, s->fl_align);
2998 else
2999 restore_rx_bufs(&si, &rxq->fl, frags);
3000 } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
3001 ret = q->handler(q, q->cur_desc, NULL);
3002 } else {
3003 ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
3004 }
3005
3006 if (unlikely(ret)) {
3007
3008 q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX);
3009 break;
3010 }
3011
3012 rspq_next(q);
3013 budget_left--;
3014 }
3015
3016 if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 16)
3017 __refill_fl(q->adap, &rxq->fl);
3018 return budget - budget_left;
3019}
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032static int napi_rx_handler(struct napi_struct *napi, int budget)
3033{
3034 unsigned int params;
3035 struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
3036 int work_done;
3037 u32 val;
3038
3039 work_done = process_responses(q, budget);
3040 if (likely(work_done < budget)) {
3041 int timer_index;
3042
3043 napi_complete_done(napi, work_done);
3044 timer_index = QINTR_TIMER_IDX_G(q->next_intr_params);
3045
3046 if (q->adaptive_rx) {
3047 if (work_done > max(timer_pkt_quota[timer_index],
3048 MIN_NAPI_WORK))
3049 timer_index = (timer_index + 1);
3050 else
3051 timer_index = timer_index - 1;
3052
3053 timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1);
3054 q->next_intr_params =
3055 QINTR_TIMER_IDX_V(timer_index) |
3056 QINTR_CNT_EN_V(0);
3057 params = q->next_intr_params;
3058 } else {
3059 params = q->next_intr_params;
3060 q->next_intr_params = q->intr_params;
3061 }
3062 } else
3063 params = QINTR_TIMER_IDX_V(7);
3064
3065 val = CIDXINC_V(work_done) | SEINTARM_V(params);
3066
3067
3068
3069
3070 if (unlikely(q->bar2_addr == NULL)) {
3071 t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
3072 val | INGRESSQID_V((u32)q->cntxt_id));
3073 } else {
3074 writel(val | INGRESSQID_V(q->bar2_qid),
3075 q->bar2_addr + SGE_UDB_GTS);
3076 wmb();
3077 }
3078 return work_done;
3079}
3080
3081
3082
3083
3084irqreturn_t t4_sge_intr_msix(int irq, void *cookie)
3085{
3086 struct sge_rspq *q = cookie;
3087
3088 napi_schedule(&q->napi);
3089 return IRQ_HANDLED;
3090}
3091
3092
3093
3094
3095
3096static unsigned int process_intrq(struct adapter *adap)
3097{
3098 unsigned int credits;
3099 const struct rsp_ctrl *rc;
3100 struct sge_rspq *q = &adap->sge.intrq;
3101 u32 val;
3102
3103 spin_lock(&adap->sge.intrq_lock);
3104 for (credits = 0; ; credits++) {
3105 rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
3106 if (!is_new_response(rc, q))
3107 break;
3108
3109 dma_rmb();
3110 if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) {
3111 unsigned int qid = ntohl(rc->pldbuflen_qid);
3112
3113 qid -= adap->sge.ingr_start;
3114 napi_schedule(&adap->sge.ingr_map[qid]->napi);
3115 }
3116
3117 rspq_next(q);
3118 }
3119
3120 val = CIDXINC_V(credits) | SEINTARM_V(q->intr_params);
3121
3122
3123
3124
3125 if (unlikely(q->bar2_addr == NULL)) {
3126 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
3127 val | INGRESSQID_V(q->cntxt_id));
3128 } else {
3129 writel(val | INGRESSQID_V(q->bar2_qid),
3130 q->bar2_addr + SGE_UDB_GTS);
3131 wmb();
3132 }
3133 spin_unlock(&adap->sge.intrq_lock);
3134 return credits;
3135}
3136
3137
3138
3139
3140
3141static irqreturn_t t4_intr_msi(int irq, void *cookie)
3142{
3143 struct adapter *adap = cookie;
3144
3145 if (adap->flags & MASTER_PF)
3146 t4_slow_intr_handler(adap);
3147 process_intrq(adap);
3148 return IRQ_HANDLED;
3149}
3150
3151
3152
3153
3154
3155
3156static irqreturn_t t4_intr_intx(int irq, void *cookie)
3157{
3158 struct adapter *adap = cookie;
3159
3160 t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0);
3161 if (((adap->flags & MASTER_PF) && t4_slow_intr_handler(adap)) |
3162 process_intrq(adap))
3163 return IRQ_HANDLED;
3164 return IRQ_NONE;
3165}
3166
3167
3168
3169
3170
3171
3172
3173
3174irq_handler_t t4_intr_handler(struct adapter *adap)
3175{
3176 if (adap->flags & USING_MSIX)
3177 return t4_sge_intr_msix;
3178 if (adap->flags & USING_MSI)
3179 return t4_intr_msi;
3180 return t4_intr_intx;
3181}
3182
3183static void sge_rx_timer_cb(struct timer_list *t)
3184{
3185 unsigned long m;
3186 unsigned int i;
3187 struct adapter *adap = from_timer(adap, t, sge.rx_timer);
3188 struct sge *s = &adap->sge;
3189
3190 for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3191 for (m = s->starving_fl[i]; m; m &= m - 1) {
3192 struct sge_eth_rxq *rxq;
3193 unsigned int id = __ffs(m) + i * BITS_PER_LONG;
3194 struct sge_fl *fl = s->egr_map[id];
3195
3196 clear_bit(id, s->starving_fl);
3197 smp_mb__after_clear_bit();
3198
3199 if (fl_starving(adap, fl)) {
3200 rxq = container_of(fl, struct sge_eth_rxq, fl);
3201 if (napi_reschedule(&rxq->rspq.napi))
3202 fl->starving++;
3203 else
3204 set_bit(id, s->starving_fl);
3205 }
3206 }
3207
3208
3209
3210
3211 if (!(adap->flags & MASTER_PF))
3212 goto done;
3213
3214 t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD);
3215
3216done:
3217 mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
3218}
3219
3220static void sge_tx_timer_cb(struct timer_list *t)
3221{
3222 unsigned long m;
3223 unsigned int i, budget;
3224 struct adapter *adap = from_timer(adap, t, sge.tx_timer);
3225 struct sge *s = &adap->sge;
3226
3227 for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3228 for (m = s->txq_maperr[i]; m; m &= m - 1) {
3229 unsigned long id = __ffs(m) + i * BITS_PER_LONG;
3230 struct sge_uld_txq *txq = s->egr_map[id];
3231
3232 clear_bit(id, s->txq_maperr);
3233 tasklet_schedule(&txq->qresume_tsk);
3234 }
3235
3236 if (!is_t4(adap->params.chip)) {
3237 struct sge_eth_txq *q = &s->ptptxq;
3238 int avail;
3239
3240 spin_lock(&adap->ptp_lock);
3241 avail = reclaimable(&q->q);
3242
3243 if (avail) {
3244 free_tx_desc(adap, &q->q, avail, false);
3245 q->q.in_use -= avail;
3246 }
3247 spin_unlock(&adap->ptp_lock);
3248 }
3249
3250 budget = MAX_TIMER_TX_RECLAIM;
3251 i = s->ethtxq_rover;
3252 do {
3253 struct sge_eth_txq *q = &s->ethtxq[i];
3254
3255 if (q->q.in_use &&
3256 time_after_eq(jiffies, q->txq->trans_start + HZ / 100) &&
3257 __netif_tx_trylock(q->txq)) {
3258 int avail = reclaimable(&q->q);
3259
3260 if (avail) {
3261 if (avail > budget)
3262 avail = budget;
3263
3264 free_tx_desc(adap, &q->q, avail, true);
3265 q->q.in_use -= avail;
3266 budget -= avail;
3267 }
3268 __netif_tx_unlock(q->txq);
3269 }
3270
3271 if (++i >= s->ethqsets)
3272 i = 0;
3273 } while (budget && i != s->ethtxq_rover);
3274 s->ethtxq_rover = i;
3275 mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
3276}
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291static void __iomem *bar2_address(struct adapter *adapter,
3292 unsigned int qid,
3293 enum t4_bar2_qtype qtype,
3294 unsigned int *pbar2_qid)
3295{
3296 u64 bar2_qoffset;
3297 int ret;
3298
3299 ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0,
3300 &bar2_qoffset, pbar2_qid);
3301 if (ret)
3302 return NULL;
3303 return adapter->bar2 + bar2_qoffset;
3304}
3305
3306
3307
3308
3309int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
3310 struct net_device *dev, int intr_idx,
3311 struct sge_fl *fl, rspq_handler_t hnd,
3312 rspq_flush_handler_t flush_hnd, int cong)
3313{
3314 int ret, flsz = 0;
3315 struct fw_iq_cmd c;
3316 struct sge *s = &adap->sge;
3317 struct port_info *pi = netdev_priv(dev);
3318 int relaxed = !(adap->flags & ROOT_NO_RELAXED_ORDERING);
3319
3320
3321 iq->size = roundup(iq->size, 16);
3322
3323 iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0,
3324 &iq->phys_addr, NULL, 0,
3325 dev_to_node(adap->pdev_dev));
3326 if (!iq->desc)
3327 return -ENOMEM;
3328
3329 memset(&c, 0, sizeof(c));
3330 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
3331 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3332 FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0));
3333 c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F |
3334 FW_LEN16(c));
3335 c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
3336 FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) |
3337 FW_IQ_CMD_IQANDST_V(intr_idx < 0) |
3338 FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) |
3339 FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx :
3340 -intr_idx - 1));
3341 c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) |
3342 FW_IQ_CMD_IQGTSMODE_F |
3343 FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) |
3344 FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4));
3345 c.iqsize = htons(iq->size);
3346 c.iqaddr = cpu_to_be64(iq->phys_addr);
3347 if (cong >= 0)
3348 c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F |
3349 FW_IQ_CMD_IQTYPE_V(cong ? FW_IQ_IQTYPE_NIC
3350 : FW_IQ_IQTYPE_OFLD));
3351
3352 if (fl) {
3353 enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
3354
3355
3356
3357
3358
3359
3360
3361
3362 if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
3363 fl->size = s->fl_starve_thres - 1 + 2 * 8;
3364 fl->size = roundup(fl->size, 8);
3365 fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64),
3366 sizeof(struct rx_sw_desc), &fl->addr,
3367 &fl->sdesc, s->stat_len,
3368 dev_to_node(adap->pdev_dev));
3369 if (!fl->desc)
3370 goto fl_nomem;
3371
3372 flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
3373 c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F |
3374 FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
3375 FW_IQ_CMD_FL0DATARO_V(relaxed) |
3376 FW_IQ_CMD_FL0PADEN_F);
3377 if (cong >= 0)
3378 c.iqns_to_fl0congen |=
3379 htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) |
3380 FW_IQ_CMD_FL0CONGCIF_F |
3381 FW_IQ_CMD_FL0CONGEN_F);
3382
3383
3384
3385
3386
3387
3388
3389
3390 c.fl0dcaen_to_fl0cidxfthresh =
3391 htons(FW_IQ_CMD_FL0FBMIN_V(chip <= CHELSIO_T5 ?
3392 FETCHBURSTMIN_128B_X :
3393 FETCHBURSTMIN_64B_X) |
3394 FW_IQ_CMD_FL0FBMAX_V((chip <= CHELSIO_T5) ?
3395 FETCHBURSTMAX_512B_X :
3396 FETCHBURSTMAX_256B_X));
3397 c.fl0size = htons(flsz);
3398 c.fl0addr = cpu_to_be64(fl->addr);
3399 }
3400
3401 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3402 if (ret)
3403 goto err;
3404
3405 netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
3406 iq->cur_desc = iq->desc;
3407 iq->cidx = 0;
3408 iq->gen = 1;
3409 iq->next_intr_params = iq->intr_params;
3410 iq->cntxt_id = ntohs(c.iqid);
3411 iq->abs_id = ntohs(c.physiqid);
3412 iq->bar2_addr = bar2_address(adap,
3413 iq->cntxt_id,
3414 T4_BAR2_QTYPE_INGRESS,
3415 &iq->bar2_qid);
3416 iq->size--;
3417 iq->netdev = dev;
3418 iq->handler = hnd;
3419 iq->flush_handler = flush_hnd;
3420
3421 memset(&iq->lro_mgr, 0, sizeof(struct t4_lro_mgr));
3422 skb_queue_head_init(&iq->lro_mgr.lroq);
3423
3424
3425 iq->offset = fl ? 0 : -1;
3426
3427 adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq;
3428
3429 if (fl) {
3430 fl->cntxt_id = ntohs(c.fl0id);
3431 fl->avail = fl->pend_cred = 0;
3432 fl->pidx = fl->cidx = 0;
3433 fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0;
3434 adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl;
3435
3436
3437
3438
3439 fl->bar2_addr = bar2_address(adap,
3440 fl->cntxt_id,
3441 T4_BAR2_QTYPE_EGRESS,
3442 &fl->bar2_qid);
3443 refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL);
3444 }
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454 if (!is_t4(adap->params.chip) && cong >= 0) {
3455 u32 param, val, ch_map = 0;
3456 int i;
3457 u16 cng_ch_bits_log = adap->params.arch.cng_ch_bits_log;
3458
3459 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3460 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3461 FW_PARAMS_PARAM_YZ_V(iq->cntxt_id));
3462 if (cong == 0) {
3463 val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X);
3464 } else {
3465 val =
3466 CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X);
3467 for (i = 0; i < 4; i++) {
3468 if (cong & (1 << i))
3469 ch_map |= 1 << (i << cng_ch_bits_log);
3470 }
3471 val |= CONMCTXT_CNGCHMAP_V(ch_map);
3472 }
3473 ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
3474 ¶m, &val);
3475 if (ret)
3476 dev_warn(adap->pdev_dev, "Failed to set Congestion"
3477 " Manager Context for Ingress Queue %d: %d\n",
3478 iq->cntxt_id, -ret);
3479 }
3480
3481 return 0;
3482
3483fl_nomem:
3484 ret = -ENOMEM;
3485err:
3486 if (iq->desc) {
3487 dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len,
3488 iq->desc, iq->phys_addr);
3489 iq->desc = NULL;
3490 }
3491 if (fl && fl->desc) {
3492 kfree(fl->sdesc);
3493 fl->sdesc = NULL;
3494 dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc),
3495 fl->desc, fl->addr);
3496 fl->desc = NULL;
3497 }
3498 return ret;
3499}
3500
3501static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id)
3502{
3503 q->cntxt_id = id;
3504 q->bar2_addr = bar2_address(adap,
3505 q->cntxt_id,
3506 T4_BAR2_QTYPE_EGRESS,
3507 &q->bar2_qid);
3508 q->in_use = 0;
3509 q->cidx = q->pidx = 0;
3510 q->stops = q->restarts = 0;
3511 q->stat = (void *)&q->desc[q->size];
3512 spin_lock_init(&q->db_lock);
3513 adap->sge.egr_map[id - adap->sge.egr_start] = q;
3514}
3515
3516int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
3517 struct net_device *dev, struct netdev_queue *netdevq,
3518 unsigned int iqid)
3519{
3520 int ret, nentries;
3521 struct fw_eq_eth_cmd c;
3522 struct sge *s = &adap->sge;
3523 struct port_info *pi = netdev_priv(dev);
3524
3525
3526 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3527
3528 txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3529 sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3530 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3531 netdev_queue_numa_node_read(netdevq));
3532 if (!txq->q.desc)
3533 return -ENOMEM;
3534
3535 memset(&c, 0, sizeof(c));
3536 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F |
3537 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3538 FW_EQ_ETH_CMD_PFN_V(adap->pf) |
3539 FW_EQ_ETH_CMD_VFN_V(0));
3540 c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F |
3541 FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c));
3542 c.viid_pkd = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
3543 FW_EQ_ETH_CMD_VIID_V(pi->viid));
3544 c.fetchszm_to_iqid =
3545 htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3546 FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) |
3547 FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid));
3548 c.dcaen_to_eqsize =
3549 htonl(FW_EQ_ETH_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3550 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3551 FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3552 FW_EQ_ETH_CMD_EQSIZE_V(nentries));
3553 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3554
3555 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3556 if (ret) {
3557 kfree(txq->q.sdesc);
3558 txq->q.sdesc = NULL;
3559 dma_free_coherent(adap->pdev_dev,
3560 nentries * sizeof(struct tx_desc),
3561 txq->q.desc, txq->q.phys_addr);
3562 txq->q.desc = NULL;
3563 return ret;
3564 }
3565
3566 txq->q.q_type = CXGB4_TXQ_ETH;
3567 init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd)));
3568 txq->txq = netdevq;
3569 txq->tso = txq->tx_cso = txq->vlan_ins = 0;
3570 txq->mapping_err = 0;
3571 return 0;
3572}
3573
3574int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
3575 struct net_device *dev, unsigned int iqid,
3576 unsigned int cmplqid)
3577{
3578 int ret, nentries;
3579 struct fw_eq_ctrl_cmd c;
3580 struct sge *s = &adap->sge;
3581 struct port_info *pi = netdev_priv(dev);
3582
3583
3584 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3585
3586 txq->q.desc = alloc_ring(adap->pdev_dev, nentries,
3587 sizeof(struct tx_desc), 0, &txq->q.phys_addr,
3588 NULL, 0, dev_to_node(adap->pdev_dev));
3589 if (!txq->q.desc)
3590 return -ENOMEM;
3591
3592 c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F |
3593 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3594 FW_EQ_CTRL_CMD_PFN_V(adap->pf) |
3595 FW_EQ_CTRL_CMD_VFN_V(0));
3596 c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F |
3597 FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c));
3598 c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid));
3599 c.physeqid_pkd = htonl(0);
3600 c.fetchszm_to_iqid =
3601 htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3602 FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) |
3603 FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid));
3604 c.dcaen_to_eqsize =
3605 htonl(FW_EQ_CTRL_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3606 FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3607 FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3608 FW_EQ_CTRL_CMD_EQSIZE_V(nentries));
3609 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3610
3611 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3612 if (ret) {
3613 dma_free_coherent(adap->pdev_dev,
3614 nentries * sizeof(struct tx_desc),
3615 txq->q.desc, txq->q.phys_addr);
3616 txq->q.desc = NULL;
3617 return ret;
3618 }
3619
3620 txq->q.q_type = CXGB4_TXQ_CTRL;
3621 init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid)));
3622 txq->adap = adap;
3623 skb_queue_head_init(&txq->sendq);
3624 tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq);
3625 txq->full = 0;
3626 return 0;
3627}
3628
3629int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
3630 unsigned int cmplqid)
3631{
3632 u32 param, val;
3633
3634 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3635 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL) |
3636 FW_PARAMS_PARAM_YZ_V(eqid));
3637 val = cmplqid;
3638 return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val);
3639}
3640
3641int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq,
3642 struct net_device *dev, unsigned int iqid,
3643 unsigned int uld_type)
3644{
3645 int ret, nentries;
3646 struct fw_eq_ofld_cmd c;
3647 struct sge *s = &adap->sge;
3648 struct port_info *pi = netdev_priv(dev);
3649 int cmd = FW_EQ_OFLD_CMD;
3650
3651
3652 nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3653
3654 txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3655 sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3656 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3657 NUMA_NO_NODE);
3658 if (!txq->q.desc)
3659 return -ENOMEM;
3660
3661 memset(&c, 0, sizeof(c));
3662 if (unlikely(uld_type == CXGB4_TX_CRYPTO))
3663 cmd = FW_EQ_CTRL_CMD;
3664 c.op_to_vfn = htonl(FW_CMD_OP_V(cmd) | FW_CMD_REQUEST_F |
3665 FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3666 FW_EQ_OFLD_CMD_PFN_V(adap->pf) |
3667 FW_EQ_OFLD_CMD_VFN_V(0));
3668 c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F |
3669 FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c));
3670 c.fetchszm_to_iqid =
3671 htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3672 FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) |
3673 FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid));
3674 c.dcaen_to_eqsize =
3675 htonl(FW_EQ_OFLD_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3676 FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3677 FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3678 FW_EQ_OFLD_CMD_EQSIZE_V(nentries));
3679 c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3680
3681 ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3682 if (ret) {
3683 kfree(txq->q.sdesc);
3684 txq->q.sdesc = NULL;
3685 dma_free_coherent(adap->pdev_dev,
3686 nentries * sizeof(struct tx_desc),
3687 txq->q.desc, txq->q.phys_addr);
3688 txq->q.desc = NULL;
3689 return ret;
3690 }
3691
3692 txq->q.q_type = CXGB4_TXQ_ULD;
3693 init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
3694 txq->adap = adap;
3695 skb_queue_head_init(&txq->sendq);
3696 tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq);
3697 txq->full = 0;
3698 txq->mapping_err = 0;
3699 return 0;
3700}
3701
3702void free_txq(struct adapter *adap, struct sge_txq *q)
3703{
3704 struct sge *s = &adap->sge;
3705
3706 dma_free_coherent(adap->pdev_dev,
3707 q->size * sizeof(struct tx_desc) + s->stat_len,
3708 q->desc, q->phys_addr);
3709 q->cntxt_id = 0;
3710 q->sdesc = NULL;
3711 q->desc = NULL;
3712}
3713
3714void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
3715 struct sge_fl *fl)
3716{
3717 struct sge *s = &adap->sge;
3718 unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
3719
3720 adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL;
3721 t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
3722 rq->cntxt_id, fl_id, 0xffff);
3723 dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len,
3724 rq->desc, rq->phys_addr);
3725 netif_napi_del(&rq->napi);
3726 rq->netdev = NULL;
3727 rq->cntxt_id = rq->abs_id = 0;
3728 rq->desc = NULL;
3729
3730 if (fl) {
3731 free_rx_bufs(adap, fl, fl->avail);
3732 dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len,
3733 fl->desc, fl->addr);
3734 kfree(fl->sdesc);
3735 fl->sdesc = NULL;
3736 fl->cntxt_id = 0;
3737 fl->desc = NULL;
3738 }
3739}
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q)
3750{
3751 for ( ; n; n--, q++)
3752 if (q->rspq.desc)
3753 free_rspq_fl(adap, &q->rspq,
3754 q->fl.size ? &q->fl : NULL);
3755}
3756
3757
3758
3759
3760
3761
3762
3763void t4_free_sge_resources(struct adapter *adap)
3764{
3765 int i;
3766 struct sge_eth_rxq *eq;
3767 struct sge_eth_txq *etq;
3768
3769
3770 for (i = 0; i < adap->sge.ethqsets; i++) {
3771 eq = &adap->sge.ethrxq[i];
3772 if (eq->rspq.desc)
3773 t4_iq_stop(adap, adap->mbox, adap->pf, 0,
3774 FW_IQ_TYPE_FL_INT_CAP,
3775 eq->rspq.cntxt_id,
3776 eq->fl.size ? eq->fl.cntxt_id : 0xffff,
3777 0xffff);
3778 }
3779
3780
3781 for (i = 0; i < adap->sge.ethqsets; i++) {
3782 eq = &adap->sge.ethrxq[i];
3783 if (eq->rspq.desc)
3784 free_rspq_fl(adap, &eq->rspq,
3785 eq->fl.size ? &eq->fl : NULL);
3786
3787 etq = &adap->sge.ethtxq[i];
3788 if (etq->q.desc) {
3789 t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
3790 etq->q.cntxt_id);
3791 __netif_tx_lock_bh(etq->txq);
3792 free_tx_desc(adap, &etq->q, etq->q.in_use, true);
3793 __netif_tx_unlock_bh(etq->txq);
3794 kfree(etq->q.sdesc);
3795 free_txq(adap, &etq->q);
3796 }
3797 }
3798
3799
3800 for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
3801 struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
3802
3803 if (cq->q.desc) {
3804 tasklet_kill(&cq->qresume_tsk);
3805 t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
3806 cq->q.cntxt_id);
3807 __skb_queue_purge(&cq->sendq);
3808 free_txq(adap, &cq->q);
3809 }
3810 }
3811
3812 if (adap->sge.fw_evtq.desc)
3813 free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
3814
3815 if (adap->sge.intrq.desc)
3816 free_rspq_fl(adap, &adap->sge.intrq, NULL);
3817
3818 if (!is_t4(adap->params.chip)) {
3819 etq = &adap->sge.ptptxq;
3820 if (etq->q.desc) {
3821 t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
3822 etq->q.cntxt_id);
3823 spin_lock_bh(&adap->ptp_lock);
3824 free_tx_desc(adap, &etq->q, etq->q.in_use, true);
3825 spin_unlock_bh(&adap->ptp_lock);
3826 kfree(etq->q.sdesc);
3827 free_txq(adap, &etq->q);
3828 }
3829 }
3830
3831
3832 memset(adap->sge.egr_map, 0,
3833 adap->sge.egr_sz * sizeof(*adap->sge.egr_map));
3834}
3835
3836void t4_sge_start(struct adapter *adap)
3837{
3838 adap->sge.ethtxq_rover = 0;
3839 mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
3840 mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
3841}
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851void t4_sge_stop(struct adapter *adap)
3852{
3853 int i;
3854 struct sge *s = &adap->sge;
3855
3856 if (in_interrupt())
3857 return;
3858
3859 if (s->rx_timer.function)
3860 del_timer_sync(&s->rx_timer);
3861 if (s->tx_timer.function)
3862 del_timer_sync(&s->tx_timer);
3863
3864 if (is_offload(adap)) {
3865 struct sge_uld_txq_info *txq_info;
3866
3867 txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
3868 if (txq_info) {
3869 struct sge_uld_txq *txq = txq_info->uldtxq;
3870
3871 for_each_ofldtxq(&adap->sge, i) {
3872 if (txq->q.desc)
3873 tasklet_kill(&txq->qresume_tsk);
3874 }
3875 }
3876 }
3877
3878 if (is_pci_uld(adap)) {
3879 struct sge_uld_txq_info *txq_info;
3880
3881 txq_info = adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
3882 if (txq_info) {
3883 struct sge_uld_txq *txq = txq_info->uldtxq;
3884
3885 for_each_ofldtxq(&adap->sge, i) {
3886 if (txq->q.desc)
3887 tasklet_kill(&txq->qresume_tsk);
3888 }
3889 }
3890 }
3891
3892 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) {
3893 struct sge_ctrl_txq *cq = &s->ctrlq[i];
3894
3895 if (cq->q.desc)
3896 tasklet_kill(&cq->qresume_tsk);
3897 }
3898}
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908static int t4_sge_init_soft(struct adapter *adap)
3909{
3910 struct sge *s = &adap->sge;
3911 u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu;
3912 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
3913 u32 ingress_rx_threshold;
3914
3915
3916
3917
3918
3919
3920 if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) !=
3921 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
3922 dev_err(adap->pdev_dev, "bad SGE CPL MODE\n");
3923 return -EINVAL;
3924 }
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934 #define READ_FL_BUF(x) \
3935 t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
3936
3937 fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF);
3938 fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF);
3939 fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF);
3940 fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF);
3941
3942
3943
3944
3945 if (fl_large_pg <= fl_small_pg)
3946 fl_large_pg = 0;
3947
3948 #undef READ_FL_BUF
3949
3950
3951
3952
3953 if (fl_small_pg != PAGE_SIZE ||
3954 (fl_large_pg & (fl_large_pg-1)) != 0) {
3955 dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n",
3956 fl_small_pg, fl_large_pg);
3957 return -EINVAL;
3958 }
3959 if (fl_large_pg)
3960 s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
3961
3962 if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) ||
3963 fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) {
3964 dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n",
3965 fl_small_mtu, fl_large_mtu);
3966 return -EINVAL;
3967 }
3968
3969
3970
3971
3972
3973 timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A);
3974 timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A);
3975 timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A);
3976 s->timer_val[0] = core_ticks_to_us(adap,
3977 TIMERVALUE0_G(timer_value_0_and_1));
3978 s->timer_val[1] = core_ticks_to_us(adap,
3979 TIMERVALUE1_G(timer_value_0_and_1));
3980 s->timer_val[2] = core_ticks_to_us(adap,
3981 TIMERVALUE2_G(timer_value_2_and_3));
3982 s->timer_val[3] = core_ticks_to_us(adap,
3983 TIMERVALUE3_G(timer_value_2_and_3));
3984 s->timer_val[4] = core_ticks_to_us(adap,
3985 TIMERVALUE4_G(timer_value_4_and_5));
3986 s->timer_val[5] = core_ticks_to_us(adap,
3987 TIMERVALUE5_G(timer_value_4_and_5));
3988
3989 ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A);
3990 s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
3991 s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
3992 s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
3993 s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
3994
3995 return 0;
3996}
3997
3998
3999
4000
4001
4002
4003
4004
4005int t4_sge_init(struct adapter *adap)
4006{
4007 struct sge *s = &adap->sge;
4008 u32 sge_control, sge_conm_ctrl;
4009 int ret, egress_threshold;
4010
4011
4012
4013
4014
4015 sge_control = t4_read_reg(adap, SGE_CONTROL_A);
4016 s->pktshift = PKTSHIFT_G(sge_control);
4017 s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;
4018
4019 s->fl_align = t4_fl_pkt_align(adap);
4020 ret = t4_sge_init_soft(adap);
4021 if (ret < 0)
4022 return ret;
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036 sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A);
4037 switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
4038 case CHELSIO_T4:
4039 egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl);
4040 break;
4041 case CHELSIO_T5:
4042 egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4043 break;
4044 case CHELSIO_T6:
4045 egress_threshold = T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4046 break;
4047 default:
4048 dev_err(adap->pdev_dev, "Unsupported Chip version %d\n",
4049 CHELSIO_CHIP_VERSION(adap->params.chip));
4050 return -EINVAL;
4051 }
4052 s->fl_starve_thres = 2*egress_threshold + 1;
4053
4054 t4_idma_monitor_init(adap, &s->idma_monitor);
4055
4056
4057
4058
4059 timer_setup(&s->rx_timer, sge_rx_timer_cb, 0);
4060 timer_setup(&s->tx_timer, sge_tx_timer_cb, 0);
4061
4062 spin_lock_init(&s->intrq_lock);
4063
4064 return 0;
4065}
4066