1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/skbuff.h>
33#include <linux/netdevice.h>
34#include <linux/etherdevice.h>
35#include <linux/if_vlan.h>
36#include <linux/ip.h>
37#include <linux/tcp.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/prefetch.h>
41#include <net/arp.h>
42#include "common.h"
43#include "regs.h"
44#include "sge_defs.h"
45#include "t3_cpl.h"
46#include "firmware_exports.h"
47#include "cxgb3_offload.h"
48
49#define USE_GTS 0
50
51#define SGE_RX_SM_BUF_SIZE 1536
52
53#define SGE_RX_COPY_THRES 256
54#define SGE_RX_PULL_LEN 128
55
56#define SGE_PG_RSVD SMP_CACHE_BYTES
57
58
59
60
61
62#define FL0_PG_CHUNK_SIZE 2048
63#define FL0_PG_ORDER 0
64#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
65#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
66#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
67#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
68
69#define SGE_RX_DROP_THRES 16
70#define RX_RECLAIM_PERIOD (HZ/4)
71
72
73
74
75#define MAX_RX_REFILL 16U
76
77
78
79
80#define TX_RECLAIM_PERIOD (HZ / 4)
81#define TX_RECLAIM_TIMER_CHUNK 64U
82#define TX_RECLAIM_CHUNK 16U
83
84
85#define WR_LEN (WR_FLITS * 8)
86
87
88
89
90enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
91
92
93enum {
94 TXQ_RUNNING = 1 << 0,
95 TXQ_LAST_PKT_DB = 1 << 1,
96};
97
98struct tx_desc {
99 __be64 flit[TX_DESC_FLITS];
100};
101
102struct rx_desc {
103 __be32 addr_lo;
104 __be32 len_gen;
105 __be32 gen2;
106 __be32 addr_hi;
107};
108
109struct tx_sw_desc {
110 struct sk_buff *skb;
111 u8 eop;
112 u8 addr_idx;
113 u8 fragidx;
114 s8 sflit;
115};
116
117struct rx_sw_desc {
118 union {
119 struct sk_buff *skb;
120 struct fl_pg_chunk pg_chunk;
121 };
122 DEFINE_DMA_UNMAP_ADDR(dma_addr);
123};
124
125struct rsp_desc {
126 struct rss_header rss_hdr;
127 __be32 flags;
128 __be32 len_cq;
129 u8 imm_data[47];
130 u8 intr_gen;
131};
132
133
134
135
136
137struct deferred_unmap_info {
138 struct pci_dev *pdev;
139 dma_addr_t addr[MAX_SKB_FRAGS + 1];
140};
141
142
143
144
145
146
147
148
149
150static u8 flit_desc_map[] = {
151 0,
152#if SGE_NUM_GENBITS == 1
153 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
154 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
155 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
156 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
157#elif SGE_NUM_GENBITS == 2
158 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
159 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
160 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
161 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
162#else
163# error "SGE_NUM_GENBITS must be 1 or 2"
164#endif
165};
166
167static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
168{
169 return container_of(q, struct sge_qset, fl[qidx]);
170}
171
172static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
173{
174 return container_of(q, struct sge_qset, rspq);
175}
176
177static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
178{
179 return container_of(q, struct sge_qset, txq[qidx]);
180}
181
182
183
184
185
186
187
188
189
190
191static inline void refill_rspq(struct adapter *adapter,
192 const struct sge_rspq *q, unsigned int credits)
193{
194 rmb();
195 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
196 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
197}
198
199
200
201
202
203
204
205static inline int need_skb_unmap(void)
206{
207#ifdef CONFIG_NEED_DMA_MAP_STATE
208 return 1;
209#else
210 return 0;
211#endif
212}
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
237 unsigned int cidx, struct pci_dev *pdev)
238{
239 const struct sg_ent *sgp;
240 struct tx_sw_desc *d = &q->sdesc[cidx];
241 int nfrags, frag_idx, curflit, j = d->addr_idx;
242
243 sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
244 frag_idx = d->fragidx;
245
246 if (frag_idx == 0 && skb_headlen(skb)) {
247 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]),
248 skb_headlen(skb), PCI_DMA_TODEVICE);
249 j = 1;
250 }
251
252 curflit = d->sflit + 1 + j;
253 nfrags = skb_shinfo(skb)->nr_frags;
254
255 while (frag_idx < nfrags && curflit < WR_FLITS) {
256 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
257 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]),
258 PCI_DMA_TODEVICE);
259 j ^= 1;
260 if (j == 0) {
261 sgp++;
262 curflit++;
263 }
264 curflit++;
265 frag_idx++;
266 }
267
268 if (frag_idx < nfrags) {
269 d = cidx + 1 == q->size ? q->sdesc : d + 1;
270 d->fragidx = frag_idx;
271 d->addr_idx = j;
272 d->sflit = curflit - WR_FLITS - j;
273 }
274}
275
276
277
278
279
280
281
282
283
284
285static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
286 unsigned int n)
287{
288 struct tx_sw_desc *d;
289 struct pci_dev *pdev = adapter->pdev;
290 unsigned int cidx = q->cidx;
291
292 const int need_unmap = need_skb_unmap() &&
293 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
294
295 d = &q->sdesc[cidx];
296 while (n--) {
297 if (d->skb) {
298 if (need_unmap)
299 unmap_skb(d->skb, q, cidx, pdev);
300 if (d->eop) {
301 dev_consume_skb_any(d->skb);
302 d->skb = NULL;
303 }
304 }
305 ++d;
306 if (++cidx == q->size) {
307 cidx = 0;
308 d = q->sdesc;
309 }
310 }
311 q->cidx = cidx;
312}
313
314
315
316
317
318
319
320
321
322
323
324static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
325 struct sge_txq *q,
326 unsigned int chunk)
327{
328 unsigned int reclaim = q->processed - q->cleaned;
329
330 reclaim = min(chunk, reclaim);
331 if (reclaim) {
332 free_tx_desc(adapter, q, reclaim);
333 q->cleaned += reclaim;
334 q->in_use -= reclaim;
335 }
336 return q->processed - q->cleaned;
337}
338
339
340
341
342
343
344
345static inline int should_restart_tx(const struct sge_txq *q)
346{
347 unsigned int r = q->processed - q->cleaned;
348
349 return q->in_use - r < (q->size >> 1);
350}
351
352static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
353 struct rx_sw_desc *d)
354{
355 if (q->use_pages && d->pg_chunk.page) {
356 (*d->pg_chunk.p_cnt)--;
357 if (!*d->pg_chunk.p_cnt)
358 pci_unmap_page(pdev,
359 d->pg_chunk.mapping,
360 q->alloc_size, PCI_DMA_FROMDEVICE);
361
362 put_page(d->pg_chunk.page);
363 d->pg_chunk.page = NULL;
364 } else {
365 pci_unmap_single(pdev, dma_unmap_addr(d, dma_addr),
366 q->buf_size, PCI_DMA_FROMDEVICE);
367 kfree_skb(d->skb);
368 d->skb = NULL;
369 }
370}
371
372
373
374
375
376
377
378
379
380static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
381{
382 unsigned int cidx = q->cidx;
383
384 while (q->credits--) {
385 struct rx_sw_desc *d = &q->sdesc[cidx];
386
387
388 clear_rx_desc(pdev, q, d);
389 if (++cidx == q->size)
390 cidx = 0;
391 }
392
393 if (q->pg_chunk.page) {
394 __free_pages(q->pg_chunk.page, q->order);
395 q->pg_chunk.page = NULL;
396 }
397}
398
399
400
401
402
403
404
405
406
407
408
409
410
411static inline int add_one_rx_buf(void *va, unsigned int len,
412 struct rx_desc *d, struct rx_sw_desc *sd,
413 unsigned int gen, struct pci_dev *pdev)
414{
415 dma_addr_t mapping;
416
417 mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
418 if (unlikely(pci_dma_mapping_error(pdev, mapping)))
419 return -ENOMEM;
420
421 dma_unmap_addr_set(sd, dma_addr, mapping);
422
423 d->addr_lo = cpu_to_be32(mapping);
424 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
425 dma_wmb();
426 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
427 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
428 return 0;
429}
430
431static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
432 unsigned int gen)
433{
434 d->addr_lo = cpu_to_be32(mapping);
435 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
436 dma_wmb();
437 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
438 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
439 return 0;
440}
441
442static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
443 struct rx_sw_desc *sd, gfp_t gfp,
444 unsigned int order)
445{
446 if (!q->pg_chunk.page) {
447 dma_addr_t mapping;
448
449 q->pg_chunk.page = alloc_pages(gfp, order);
450 if (unlikely(!q->pg_chunk.page))
451 return -ENOMEM;
452 q->pg_chunk.va = page_address(q->pg_chunk.page);
453 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
454 SGE_PG_RSVD;
455 q->pg_chunk.offset = 0;
456 mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
457 0, q->alloc_size, PCI_DMA_FROMDEVICE);
458 if (unlikely(pci_dma_mapping_error(adapter->pdev, mapping))) {
459 __free_pages(q->pg_chunk.page, order);
460 q->pg_chunk.page = NULL;
461 return -EIO;
462 }
463 q->pg_chunk.mapping = mapping;
464 }
465 sd->pg_chunk = q->pg_chunk;
466
467 prefetch(sd->pg_chunk.p_cnt);
468
469 q->pg_chunk.offset += q->buf_size;
470 if (q->pg_chunk.offset == (PAGE_SIZE << order))
471 q->pg_chunk.page = NULL;
472 else {
473 q->pg_chunk.va += q->buf_size;
474 get_page(q->pg_chunk.page);
475 }
476
477 if (sd->pg_chunk.offset == 0)
478 *sd->pg_chunk.p_cnt = 1;
479 else
480 *sd->pg_chunk.p_cnt += 1;
481
482 return 0;
483}
484
485static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
486{
487 if (q->pend_cred >= q->credits / 4) {
488 q->pend_cred = 0;
489 wmb();
490 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
491 }
492}
493
494
495
496
497
498
499
500
501
502
503
504
505static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
506{
507 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
508 struct rx_desc *d = &q->desc[q->pidx];
509 unsigned int count = 0;
510
511 while (n--) {
512 dma_addr_t mapping;
513 int err;
514
515 if (q->use_pages) {
516 if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
517 q->order))) {
518nomem: q->alloc_failed++;
519 break;
520 }
521 mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
522 dma_unmap_addr_set(sd, dma_addr, mapping);
523
524 add_one_rx_chunk(mapping, d, q->gen);
525 pci_dma_sync_single_for_device(adap->pdev, mapping,
526 q->buf_size - SGE_PG_RSVD,
527 PCI_DMA_FROMDEVICE);
528 } else {
529 void *buf_start;
530
531 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
532 if (!skb)
533 goto nomem;
534
535 sd->skb = skb;
536 buf_start = skb->data;
537 err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
538 q->gen, adap->pdev);
539 if (unlikely(err)) {
540 clear_rx_desc(adap->pdev, q, sd);
541 break;
542 }
543 }
544
545 d++;
546 sd++;
547 if (++q->pidx == q->size) {
548 q->pidx = 0;
549 q->gen ^= 1;
550 sd = q->sdesc;
551 d = q->desc;
552 }
553 count++;
554 }
555
556 q->credits += count;
557 q->pend_cred += count;
558 ring_fl_db(adap, q);
559
560 return count;
561}
562
563static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
564{
565 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
566 GFP_ATOMIC | __GFP_COMP);
567}
568
569
570
571
572
573
574
575
576
577
578static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
579 unsigned int idx)
580{
581 struct rx_desc *from = &q->desc[idx];
582 struct rx_desc *to = &q->desc[q->pidx];
583
584 q->sdesc[q->pidx] = q->sdesc[idx];
585 to->addr_lo = from->addr_lo;
586 to->addr_hi = from->addr_hi;
587 dma_wmb();
588 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
589 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
590
591 if (++q->pidx == q->size) {
592 q->pidx = 0;
593 q->gen ^= 1;
594 }
595
596 q->credits++;
597 q->pend_cred++;
598 ring_fl_db(adap, q);
599}
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
619 size_t sw_size, dma_addr_t * phys, void *metadata)
620{
621 size_t len = nelem * elem_size;
622 void *s = NULL;
623 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
624
625 if (!p)
626 return NULL;
627 if (sw_size && metadata) {
628 s = kcalloc(nelem, sw_size, GFP_KERNEL);
629
630 if (!s) {
631 dma_free_coherent(&pdev->dev, len, p, *phys);
632 return NULL;
633 }
634 *(void **)metadata = s;
635 }
636 return p;
637}
638
639
640
641
642
643
644
645
646
647static void t3_reset_qset(struct sge_qset *q)
648{
649 if (q->adap &&
650 !(q->adap->flags & NAPI_INIT)) {
651 memset(q, 0, sizeof(*q));
652 return;
653 }
654
655 q->adap = NULL;
656 memset(&q->rspq, 0, sizeof(q->rspq));
657 memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
658 memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
659 q->txq_stopped = 0;
660 q->tx_reclaim_timer.function = NULL;
661 q->rx_reclaim_timer.function = NULL;
662 q->nomem = 0;
663 napi_free_frags(&q->napi);
664}
665
666
667
668
669
670
671
672
673
674
675
676static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
677{
678 int i;
679 struct pci_dev *pdev = adapter->pdev;
680
681 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
682 if (q->fl[i].desc) {
683 spin_lock_irq(&adapter->sge.reg_lock);
684 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
685 spin_unlock_irq(&adapter->sge.reg_lock);
686 free_rx_bufs(pdev, &q->fl[i]);
687 kfree(q->fl[i].sdesc);
688 dma_free_coherent(&pdev->dev,
689 q->fl[i].size *
690 sizeof(struct rx_desc), q->fl[i].desc,
691 q->fl[i].phys_addr);
692 }
693
694 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
695 if (q->txq[i].desc) {
696 spin_lock_irq(&adapter->sge.reg_lock);
697 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
698 spin_unlock_irq(&adapter->sge.reg_lock);
699 if (q->txq[i].sdesc) {
700 free_tx_desc(adapter, &q->txq[i],
701 q->txq[i].in_use);
702 kfree(q->txq[i].sdesc);
703 }
704 dma_free_coherent(&pdev->dev,
705 q->txq[i].size *
706 sizeof(struct tx_desc),
707 q->txq[i].desc, q->txq[i].phys_addr);
708 __skb_queue_purge(&q->txq[i].sendq);
709 }
710
711 if (q->rspq.desc) {
712 spin_lock_irq(&adapter->sge.reg_lock);
713 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
714 spin_unlock_irq(&adapter->sge.reg_lock);
715 dma_free_coherent(&pdev->dev,
716 q->rspq.size * sizeof(struct rsp_desc),
717 q->rspq.desc, q->rspq.phys_addr);
718 }
719
720 t3_reset_qset(q);
721}
722
723
724
725
726
727
728
729
730static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
731{
732 qs->rspq.cntxt_id = id;
733 qs->fl[0].cntxt_id = 2 * id;
734 qs->fl[1].cntxt_id = 2 * id + 1;
735 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
736 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
737 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
738 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
739 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
740}
741
742
743
744
745
746
747
748
749static inline unsigned int sgl_len(unsigned int n)
750{
751
752 return (3 * n) / 2 + (n & 1);
753}
754
755
756
757
758
759
760
761
762static inline unsigned int flits_to_desc(unsigned int n)
763{
764 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
765 return flit_desc_map[n];
766}
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
784 unsigned int len, unsigned int drop_thres)
785{
786 struct sk_buff *skb = NULL;
787 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
788
789 prefetch(sd->skb->data);
790 fl->credits--;
791
792 if (len <= SGE_RX_COPY_THRES) {
793 skb = alloc_skb(len, GFP_ATOMIC);
794 if (likely(skb != NULL)) {
795 __skb_put(skb, len);
796 pci_dma_sync_single_for_cpu(adap->pdev,
797 dma_unmap_addr(sd, dma_addr), len,
798 PCI_DMA_FROMDEVICE);
799 memcpy(skb->data, sd->skb->data, len);
800 pci_dma_sync_single_for_device(adap->pdev,
801 dma_unmap_addr(sd, dma_addr), len,
802 PCI_DMA_FROMDEVICE);
803 } else if (!drop_thres)
804 goto use_orig_buf;
805recycle:
806 recycle_rx_buf(adap, fl, fl->cidx);
807 return skb;
808 }
809
810 if (unlikely(fl->credits < drop_thres) &&
811 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
812 GFP_ATOMIC | __GFP_COMP) == 0)
813 goto recycle;
814
815use_orig_buf:
816 pci_unmap_single(adap->pdev, dma_unmap_addr(sd, dma_addr),
817 fl->buf_size, PCI_DMA_FROMDEVICE);
818 skb = sd->skb;
819 skb_put(skb, len);
820 __refill_fl(adap, fl);
821 return skb;
822}
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
844 struct sge_rspq *q, unsigned int len,
845 unsigned int drop_thres)
846{
847 struct sk_buff *newskb, *skb;
848 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
849
850 dma_addr_t dma_addr = dma_unmap_addr(sd, dma_addr);
851
852 newskb = skb = q->pg_skb;
853 if (!skb && (len <= SGE_RX_COPY_THRES)) {
854 newskb = alloc_skb(len, GFP_ATOMIC);
855 if (likely(newskb != NULL)) {
856 __skb_put(newskb, len);
857 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
858 PCI_DMA_FROMDEVICE);
859 memcpy(newskb->data, sd->pg_chunk.va, len);
860 pci_dma_sync_single_for_device(adap->pdev, dma_addr,
861 len,
862 PCI_DMA_FROMDEVICE);
863 } else if (!drop_thres)
864 return NULL;
865recycle:
866 fl->credits--;
867 recycle_rx_buf(adap, fl, fl->cidx);
868 q->rx_recycle_buf++;
869 return newskb;
870 }
871
872 if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
873 goto recycle;
874
875 prefetch(sd->pg_chunk.p_cnt);
876
877 if (!skb)
878 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
879
880 if (unlikely(!newskb)) {
881 if (!drop_thres)
882 return NULL;
883 goto recycle;
884 }
885
886 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
887 PCI_DMA_FROMDEVICE);
888 (*sd->pg_chunk.p_cnt)--;
889 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
890 pci_unmap_page(adap->pdev,
891 sd->pg_chunk.mapping,
892 fl->alloc_size,
893 PCI_DMA_FROMDEVICE);
894 if (!skb) {
895 __skb_put(newskb, SGE_RX_PULL_LEN);
896 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
897 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
898 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
899 len - SGE_RX_PULL_LEN);
900 newskb->len = len;
901 newskb->data_len = len - SGE_RX_PULL_LEN;
902 newskb->truesize += newskb->data_len;
903 } else {
904 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
905 sd->pg_chunk.page,
906 sd->pg_chunk.offset, len);
907 newskb->len += len;
908 newskb->data_len += len;
909 newskb->truesize += len;
910 }
911
912 fl->credits--;
913
914
915
916
917 return newskb;
918}
919
920
921
922
923
924
925
926static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
927{
928 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
929
930 if (skb) {
931 __skb_put(skb, IMMED_PKT_SIZE);
932 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
933 }
934 return skb;
935}
936
937
938
939
940
941
942
943
944static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
945{
946 unsigned int flits;
947
948 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
949 return 1;
950
951 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
952 if (skb_shinfo(skb)->gso_size)
953 flits++;
954 return flits_to_desc(flits);
955}
956
957
958
959
960
961
962
963
964static int map_skb(struct pci_dev *pdev, const struct sk_buff *skb,
965 dma_addr_t *addr)
966{
967 const skb_frag_t *fp, *end;
968 const struct skb_shared_info *si;
969
970 if (skb_headlen(skb)) {
971 *addr = pci_map_single(pdev, skb->data, skb_headlen(skb),
972 PCI_DMA_TODEVICE);
973 if (pci_dma_mapping_error(pdev, *addr))
974 goto out_err;
975 addr++;
976 }
977
978 si = skb_shinfo(skb);
979 end = &si->frags[si->nr_frags];
980
981 for (fp = si->frags; fp < end; fp++) {
982 *addr = skb_frag_dma_map(&pdev->dev, fp, 0, skb_frag_size(fp),
983 DMA_TO_DEVICE);
984 if (pci_dma_mapping_error(pdev, *addr))
985 goto unwind;
986 addr++;
987 }
988 return 0;
989
990unwind:
991 while (fp-- > si->frags)
992 dma_unmap_page(&pdev->dev, *--addr, skb_frag_size(fp),
993 DMA_TO_DEVICE);
994
995 pci_unmap_single(pdev, addr[-1], skb_headlen(skb), PCI_DMA_TODEVICE);
996out_err:
997 return -ENOMEM;
998}
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012static inline unsigned int write_sgl(const struct sk_buff *skb,
1013 struct sg_ent *sgp, unsigned char *start,
1014 unsigned int len, const dma_addr_t *addr)
1015{
1016 unsigned int i, j = 0, k = 0, nfrags;
1017
1018 if (len) {
1019 sgp->len[0] = cpu_to_be32(len);
1020 sgp->addr[j++] = cpu_to_be64(addr[k++]);
1021 }
1022
1023 nfrags = skb_shinfo(skb)->nr_frags;
1024 for (i = 0; i < nfrags; i++) {
1025 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1026
1027 sgp->len[j] = cpu_to_be32(skb_frag_size(frag));
1028 sgp->addr[j] = cpu_to_be64(addr[k++]);
1029 j ^= 1;
1030 if (j == 0)
1031 ++sgp;
1032 }
1033 if (j)
1034 sgp->len[j] = 0;
1035 return ((nfrags + (len != 0)) * 3) / 2 + j;
1036}
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1051{
1052#if USE_GTS
1053 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1054 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1055 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1056 t3_write_reg(adap, A_SG_KDOORBELL,
1057 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1058 }
1059#else
1060 wmb();
1061 t3_write_reg(adap, A_SG_KDOORBELL,
1062 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1063#endif
1064}
1065
1066static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1067{
1068#if SGE_NUM_GENBITS == 2
1069 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1070#endif
1071}
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1093 struct tx_desc *d, unsigned int pidx,
1094 const struct sge_txq *q,
1095 const struct sg_ent *sgl,
1096 unsigned int flits, unsigned int sgl_flits,
1097 unsigned int gen, __be32 wr_hi,
1098 __be32 wr_lo)
1099{
1100 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1101 struct tx_sw_desc *sd = &q->sdesc[pidx];
1102
1103 sd->skb = skb;
1104 if (need_skb_unmap()) {
1105 sd->fragidx = 0;
1106 sd->addr_idx = 0;
1107 sd->sflit = flits;
1108 }
1109
1110 if (likely(ndesc == 1)) {
1111 sd->eop = 1;
1112 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1113 V_WR_SGLSFLT(flits)) | wr_hi;
1114 dma_wmb();
1115 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1116 V_WR_GEN(gen)) | wr_lo;
1117 wr_gen2(d, gen);
1118 } else {
1119 unsigned int ogen = gen;
1120 const u64 *fp = (const u64 *)sgl;
1121 struct work_request_hdr *wp = wrp;
1122
1123 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1124 V_WR_SGLSFLT(flits)) | wr_hi;
1125
1126 while (sgl_flits) {
1127 unsigned int avail = WR_FLITS - flits;
1128
1129 if (avail > sgl_flits)
1130 avail = sgl_flits;
1131 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1132 sgl_flits -= avail;
1133 ndesc--;
1134 if (!sgl_flits)
1135 break;
1136
1137 fp += avail;
1138 d++;
1139 sd->eop = 0;
1140 sd++;
1141 if (++pidx == q->size) {
1142 pidx = 0;
1143 gen ^= 1;
1144 d = q->desc;
1145 sd = q->sdesc;
1146 }
1147
1148 sd->skb = skb;
1149 wrp = (struct work_request_hdr *)d;
1150 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1151 V_WR_SGLSFLT(1)) | wr_hi;
1152 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1153 sgl_flits + 1)) |
1154 V_WR_GEN(gen)) | wr_lo;
1155 wr_gen2(d, gen);
1156 flits = 1;
1157 }
1158 sd->eop = 1;
1159 wrp->wr_hi |= htonl(F_WR_EOP);
1160 dma_wmb();
1161 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1162 wr_gen2((struct tx_desc *)wp, ogen);
1163 WARN_ON(ndesc != 0);
1164 }
1165}
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1182 const struct port_info *pi,
1183 unsigned int pidx, unsigned int gen,
1184 struct sge_txq *q, unsigned int ndesc,
1185 unsigned int compl, const dma_addr_t *addr)
1186{
1187 unsigned int flits, sgl_flits, cntrl, tso_info;
1188 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1189 struct tx_desc *d = &q->desc[pidx];
1190 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1191
1192 cpl->len = htonl(skb->len);
1193 cntrl = V_TXPKT_INTF(pi->port_id);
1194
1195 if (skb_vlan_tag_present(skb))
1196 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(skb_vlan_tag_get(skb));
1197
1198 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1199 if (tso_info) {
1200 int eth_type;
1201 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1202
1203 d->flit[2] = 0;
1204 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1205 hdr->cntrl = htonl(cntrl);
1206 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1207 CPL_ETH_II : CPL_ETH_II_VLAN;
1208 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1209 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1210 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1211 hdr->lso_info = htonl(tso_info);
1212 flits = 3;
1213 } else {
1214 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1215 cntrl |= F_TXPKT_IPCSUM_DIS;
1216 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1217 cpl->cntrl = htonl(cntrl);
1218
1219 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1220 q->sdesc[pidx].skb = NULL;
1221 if (!skb->data_len)
1222 skb_copy_from_linear_data(skb, &d->flit[2],
1223 skb->len);
1224 else
1225 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1226
1227 flits = (skb->len + 7) / 8 + 2;
1228 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1229 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1230 | F_WR_SOP | F_WR_EOP | compl);
1231 dma_wmb();
1232 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1233 V_WR_TID(q->token));
1234 wr_gen2(d, gen);
1235 dev_consume_skb_any(skb);
1236 return;
1237 }
1238
1239 flits = 2;
1240 }
1241
1242 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1243 sgl_flits = write_sgl(skb, sgp, skb->data, skb_headlen(skb), addr);
1244
1245 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1246 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1247 htonl(V_WR_TID(q->token)));
1248}
1249
1250static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1251 struct sge_qset *qs, struct sge_txq *q)
1252{
1253 netif_tx_stop_queue(txq);
1254 set_bit(TXQ_ETH, &qs->txq_stopped);
1255 q->stops++;
1256}
1257
1258
1259
1260
1261
1262
1263
1264
1265netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1266{
1267 int qidx;
1268 unsigned int ndesc, pidx, credits, gen, compl;
1269 const struct port_info *pi = netdev_priv(dev);
1270 struct adapter *adap = pi->adapter;
1271 struct netdev_queue *txq;
1272 struct sge_qset *qs;
1273 struct sge_txq *q;
1274 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1275
1276
1277
1278
1279
1280 if (unlikely(skb->len < ETH_HLEN)) {
1281 dev_kfree_skb_any(skb);
1282 return NETDEV_TX_OK;
1283 }
1284
1285 qidx = skb_get_queue_mapping(skb);
1286 qs = &pi->qs[qidx];
1287 q = &qs->txq[TXQ_ETH];
1288 txq = netdev_get_tx_queue(dev, qidx);
1289
1290 reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1291
1292 credits = q->size - q->in_use;
1293 ndesc = calc_tx_descs(skb);
1294
1295 if (unlikely(credits < ndesc)) {
1296 t3_stop_tx_queue(txq, qs, q);
1297 dev_err(&adap->pdev->dev,
1298 "%s: Tx ring %u full while queue awake!\n",
1299 dev->name, q->cntxt_id & 7);
1300 return NETDEV_TX_BUSY;
1301 }
1302
1303
1304 if (skb->len > (WR_LEN - sizeof(struct cpl_tx_pkt))) {
1305 if (unlikely(map_skb(adap->pdev, skb, addr) < 0)) {
1306 dev_kfree_skb(skb);
1307 return NETDEV_TX_OK;
1308 }
1309 }
1310
1311 q->in_use += ndesc;
1312 if (unlikely(credits - ndesc < q->stop_thres)) {
1313 t3_stop_tx_queue(txq, qs, q);
1314
1315 if (should_restart_tx(q) &&
1316 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1317 q->restarts++;
1318 netif_tx_start_queue(txq);
1319 }
1320 }
1321
1322 gen = q->gen;
1323 q->unacked += ndesc;
1324 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1325 q->unacked &= 7;
1326 pidx = q->pidx;
1327 q->pidx += ndesc;
1328 if (q->pidx >= q->size) {
1329 q->pidx -= q->size;
1330 q->gen ^= 1;
1331 }
1332
1333
1334 if (skb->ip_summed == CHECKSUM_PARTIAL)
1335 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1336 if (skb_shinfo(skb)->gso_size)
1337 qs->port_stats[SGE_PSTAT_TSO]++;
1338 if (skb_vlan_tag_present(skb))
1339 qs->port_stats[SGE_PSTAT_VLANINS]++;
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365 if (likely(!skb_shared(skb)))
1366 skb_orphan(skb);
1367
1368 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl, addr);
1369 check_ring_tx_db(adap, q);
1370 return NETDEV_TX_OK;
1371}
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1386 unsigned int len, unsigned int gen)
1387{
1388 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1389 struct work_request_hdr *to = (struct work_request_hdr *)d;
1390
1391 if (likely(!skb->data_len))
1392 memcpy(&to[1], &from[1], len - sizeof(*from));
1393 else
1394 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1395
1396 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1397 V_WR_BCNTLFLT(len & 7));
1398 dma_wmb();
1399 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1400 V_WR_LEN((len + 7) / 8));
1401 wr_gen2(d, gen);
1402 kfree_skb(skb);
1403}
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1424 struct sk_buff *skb, unsigned int ndesc,
1425 unsigned int qid)
1426{
1427 if (unlikely(!skb_queue_empty(&q->sendq))) {
1428 addq_exit:__skb_queue_tail(&q->sendq, skb);
1429 return 1;
1430 }
1431 if (unlikely(q->size - q->in_use < ndesc)) {
1432 struct sge_qset *qs = txq_to_qset(q, qid);
1433
1434 set_bit(qid, &qs->txq_stopped);
1435 smp_mb__after_atomic();
1436
1437 if (should_restart_tx(q) &&
1438 test_and_clear_bit(qid, &qs->txq_stopped))
1439 return 2;
1440
1441 q->stops++;
1442 goto addq_exit;
1443 }
1444 return 0;
1445}
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1456{
1457 unsigned int reclaim = q->processed - q->cleaned;
1458
1459 q->in_use -= reclaim;
1460 q->cleaned += reclaim;
1461}
1462
1463static inline int immediate(const struct sk_buff *skb)
1464{
1465 return skb->len <= WR_LEN;
1466}
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1479 struct sk_buff *skb)
1480{
1481 int ret;
1482 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1483
1484 if (unlikely(!immediate(skb))) {
1485 WARN_ON(1);
1486 dev_kfree_skb(skb);
1487 return NET_XMIT_SUCCESS;
1488 }
1489
1490 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1491 wrp->wr_lo = htonl(V_WR_TID(q->token));
1492
1493 spin_lock(&q->lock);
1494 again:reclaim_completed_tx_imm(q);
1495
1496 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1497 if (unlikely(ret)) {
1498 if (ret == 1) {
1499 spin_unlock(&q->lock);
1500 return NET_XMIT_CN;
1501 }
1502 goto again;
1503 }
1504
1505 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1506
1507 q->in_use++;
1508 if (++q->pidx >= q->size) {
1509 q->pidx = 0;
1510 q->gen ^= 1;
1511 }
1512 spin_unlock(&q->lock);
1513 wmb();
1514 t3_write_reg(adap, A_SG_KDOORBELL,
1515 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1516 return NET_XMIT_SUCCESS;
1517}
1518
1519
1520
1521
1522
1523
1524
1525static void restart_ctrlq(struct work_struct *w)
1526{
1527 struct sk_buff *skb;
1528 struct sge_qset *qs = container_of(w, struct sge_qset,
1529 txq[TXQ_CTRL].qresume_task);
1530 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1531
1532 spin_lock(&q->lock);
1533 again:reclaim_completed_tx_imm(q);
1534
1535 while (q->in_use < q->size &&
1536 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1537
1538 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1539
1540 if (++q->pidx >= q->size) {
1541 q->pidx = 0;
1542 q->gen ^= 1;
1543 }
1544 q->in_use++;
1545 }
1546
1547 if (!skb_queue_empty(&q->sendq)) {
1548 set_bit(TXQ_CTRL, &qs->txq_stopped);
1549 smp_mb__after_atomic();
1550
1551 if (should_restart_tx(q) &&
1552 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1553 goto again;
1554 q->stops++;
1555 }
1556
1557 spin_unlock(&q->lock);
1558 wmb();
1559 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1560 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1561}
1562
1563
1564
1565
1566int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1567{
1568 int ret;
1569 local_bh_disable();
1570 ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1571 local_bh_enable();
1572
1573 return ret;
1574}
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584static void deferred_unmap_destructor(struct sk_buff *skb)
1585{
1586 int i;
1587 const dma_addr_t *p;
1588 const struct skb_shared_info *si;
1589 const struct deferred_unmap_info *dui;
1590
1591 dui = (struct deferred_unmap_info *)skb->head;
1592 p = dui->addr;
1593
1594 if (skb_tail_pointer(skb) - skb_transport_header(skb))
1595 pci_unmap_single(dui->pdev, *p++, skb_tail_pointer(skb) -
1596 skb_transport_header(skb), PCI_DMA_TODEVICE);
1597
1598 si = skb_shinfo(skb);
1599 for (i = 0; i < si->nr_frags; i++)
1600 pci_unmap_page(dui->pdev, *p++, skb_frag_size(&si->frags[i]),
1601 PCI_DMA_TODEVICE);
1602}
1603
1604static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1605 const struct sg_ent *sgl, int sgl_flits)
1606{
1607 dma_addr_t *p;
1608 struct deferred_unmap_info *dui;
1609
1610 dui = (struct deferred_unmap_info *)skb->head;
1611 dui->pdev = pdev;
1612 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1613 *p++ = be64_to_cpu(sgl->addr[0]);
1614 *p++ = be64_to_cpu(sgl->addr[1]);
1615 }
1616 if (sgl_flits)
1617 *p = be64_to_cpu(sgl->addr[0]);
1618}
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1634 struct sge_txq *q, unsigned int pidx,
1635 unsigned int gen, unsigned int ndesc,
1636 const dma_addr_t *addr)
1637{
1638 unsigned int sgl_flits, flits;
1639 struct work_request_hdr *from;
1640 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1641 struct tx_desc *d = &q->desc[pidx];
1642
1643 if (immediate(skb)) {
1644 q->sdesc[pidx].skb = NULL;
1645 write_imm(d, skb, skb->len, gen);
1646 return;
1647 }
1648
1649
1650
1651 from = (struct work_request_hdr *)skb->data;
1652 memcpy(&d->flit[1], &from[1],
1653 skb_transport_offset(skb) - sizeof(*from));
1654
1655 flits = skb_transport_offset(skb) / 8;
1656 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1657 sgl_flits = write_sgl(skb, sgp, skb_transport_header(skb),
1658 skb_tail_pointer(skb) - skb_transport_header(skb),
1659 addr);
1660 if (need_skb_unmap()) {
1661 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1662 skb->destructor = deferred_unmap_destructor;
1663 }
1664
1665 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1666 gen, from->wr_hi, from->wr_lo);
1667}
1668
1669
1670
1671
1672
1673
1674
1675
1676static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1677{
1678 unsigned int flits, cnt;
1679
1680 if (skb->len <= WR_LEN)
1681 return 1;
1682
1683 flits = skb_transport_offset(skb) / 8;
1684 cnt = skb_shinfo(skb)->nr_frags;
1685 if (skb_tail_pointer(skb) != skb_transport_header(skb))
1686 cnt++;
1687 return flits_to_desc(flits + sgl_len(cnt));
1688}
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1699 struct sk_buff *skb)
1700{
1701 int ret;
1702 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1703
1704 spin_lock(&q->lock);
1705again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1706
1707 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1708 if (unlikely(ret)) {
1709 if (ret == 1) {
1710 skb->priority = ndesc;
1711 spin_unlock(&q->lock);
1712 return NET_XMIT_CN;
1713 }
1714 goto again;
1715 }
1716
1717 if (!immediate(skb) &&
1718 map_skb(adap->pdev, skb, (dma_addr_t *)skb->head)) {
1719 spin_unlock(&q->lock);
1720 return NET_XMIT_SUCCESS;
1721 }
1722
1723 gen = q->gen;
1724 q->in_use += ndesc;
1725 pidx = q->pidx;
1726 q->pidx += ndesc;
1727 if (q->pidx >= q->size) {
1728 q->pidx -= q->size;
1729 q->gen ^= 1;
1730 }
1731 spin_unlock(&q->lock);
1732
1733 write_ofld_wr(adap, skb, q, pidx, gen, ndesc, (dma_addr_t *)skb->head);
1734 check_ring_tx_db(adap, q);
1735 return NET_XMIT_SUCCESS;
1736}
1737
1738
1739
1740
1741
1742
1743
1744static void restart_offloadq(struct work_struct *w)
1745{
1746 struct sk_buff *skb;
1747 struct sge_qset *qs = container_of(w, struct sge_qset,
1748 txq[TXQ_OFLD].qresume_task);
1749 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1750 const struct port_info *pi = netdev_priv(qs->netdev);
1751 struct adapter *adap = pi->adapter;
1752 unsigned int written = 0;
1753
1754 spin_lock(&q->lock);
1755again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1756
1757 while ((skb = skb_peek(&q->sendq)) != NULL) {
1758 unsigned int gen, pidx;
1759 unsigned int ndesc = skb->priority;
1760
1761 if (unlikely(q->size - q->in_use < ndesc)) {
1762 set_bit(TXQ_OFLD, &qs->txq_stopped);
1763 smp_mb__after_atomic();
1764
1765 if (should_restart_tx(q) &&
1766 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1767 goto again;
1768 q->stops++;
1769 break;
1770 }
1771
1772 if (!immediate(skb) &&
1773 map_skb(adap->pdev, skb, (dma_addr_t *)skb->head))
1774 break;
1775
1776 gen = q->gen;
1777 q->in_use += ndesc;
1778 pidx = q->pidx;
1779 q->pidx += ndesc;
1780 written += ndesc;
1781 if (q->pidx >= q->size) {
1782 q->pidx -= q->size;
1783 q->gen ^= 1;
1784 }
1785 __skb_unlink(skb, &q->sendq);
1786 spin_unlock(&q->lock);
1787
1788 write_ofld_wr(adap, skb, q, pidx, gen, ndesc,
1789 (dma_addr_t *)skb->head);
1790 spin_lock(&q->lock);
1791 }
1792 spin_unlock(&q->lock);
1793
1794#if USE_GTS
1795 set_bit(TXQ_RUNNING, &q->flags);
1796 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1797#endif
1798 wmb();
1799 if (likely(written))
1800 t3_write_reg(adap, A_SG_KDOORBELL,
1801 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1802}
1803
1804
1805
1806
1807
1808
1809
1810
1811static inline int queue_set(const struct sk_buff *skb)
1812{
1813 return skb->priority >> 1;
1814}
1815
1816
1817
1818
1819
1820
1821
1822
1823static inline int is_ctrl_pkt(const struct sk_buff *skb)
1824{
1825 return skb->priority & 1;
1826}
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1838{
1839 struct adapter *adap = tdev2adap(tdev);
1840 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1841
1842 if (unlikely(is_ctrl_pkt(skb)))
1843 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1844
1845 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1846}
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1858{
1859 int was_empty = skb_queue_empty(&q->rx_queue);
1860
1861 __skb_queue_tail(&q->rx_queue, skb);
1862
1863 if (was_empty) {
1864 struct sge_qset *qs = rspq_to_qset(q);
1865
1866 napi_schedule(&qs->napi);
1867 }
1868}
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879static inline void deliver_partial_bundle(struct t3cdev *tdev,
1880 struct sge_rspq *q,
1881 struct sk_buff *skbs[], int n)
1882{
1883 if (n) {
1884 q->offload_bundles++;
1885 tdev->recv(tdev, skbs, n);
1886 }
1887}
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900static int ofld_poll(struct napi_struct *napi, int budget)
1901{
1902 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1903 struct sge_rspq *q = &qs->rspq;
1904 struct adapter *adapter = qs->adap;
1905 int work_done = 0;
1906
1907 while (work_done < budget) {
1908 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1909 struct sk_buff_head queue;
1910 int ngathered;
1911
1912 spin_lock_irq(&q->lock);
1913 __skb_queue_head_init(&queue);
1914 skb_queue_splice_init(&q->rx_queue, &queue);
1915 if (skb_queue_empty(&queue)) {
1916 napi_complete_done(napi, work_done);
1917 spin_unlock_irq(&q->lock);
1918 return work_done;
1919 }
1920 spin_unlock_irq(&q->lock);
1921
1922 ngathered = 0;
1923 skb_queue_walk_safe(&queue, skb, tmp) {
1924 if (work_done >= budget)
1925 break;
1926 work_done++;
1927
1928 __skb_unlink(skb, &queue);
1929 prefetch(skb->data);
1930 skbs[ngathered] = skb;
1931 if (++ngathered == RX_BUNDLE_SIZE) {
1932 q->offload_bundles++;
1933 adapter->tdev.recv(&adapter->tdev, skbs,
1934 ngathered);
1935 ngathered = 0;
1936 }
1937 }
1938 if (!skb_queue_empty(&queue)) {
1939
1940 spin_lock_irq(&q->lock);
1941 skb_queue_splice(&queue, &q->rx_queue);
1942 spin_unlock_irq(&q->lock);
1943 }
1944 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1945 }
1946
1947 return work_done;
1948}
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1962 struct sk_buff *skb, struct sk_buff *rx_gather[],
1963 unsigned int gather_idx)
1964{
1965 skb_reset_mac_header(skb);
1966 skb_reset_network_header(skb);
1967 skb_reset_transport_header(skb);
1968
1969 if (rq->polling) {
1970 rx_gather[gather_idx++] = skb;
1971 if (gather_idx == RX_BUNDLE_SIZE) {
1972 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1973 gather_idx = 0;
1974 rq->offload_bundles++;
1975 }
1976 } else
1977 offload_enqueue(rq, skb);
1978
1979 return gather_idx;
1980}
1981
1982
1983
1984
1985
1986
1987
1988
1989static void restart_tx(struct sge_qset *qs)
1990{
1991 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1992 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1993 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1994 qs->txq[TXQ_ETH].restarts++;
1995 if (netif_running(qs->netdev))
1996 netif_tx_wake_queue(qs->tx_q);
1997 }
1998
1999 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
2000 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
2001 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
2002 qs->txq[TXQ_OFLD].restarts++;
2003
2004
2005 queue_work(cxgb3_wq, &qs->txq[TXQ_OFLD].qresume_task);
2006 }
2007 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
2008 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
2009 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
2010 qs->txq[TXQ_CTRL].restarts++;
2011
2012
2013 queue_work(cxgb3_wq, &qs->txq[TXQ_CTRL].qresume_task);
2014 }
2015}
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb)
2026{
2027 struct net_device *dev = skb->dev;
2028 struct arphdr *arp;
2029 unsigned char *arp_ptr;
2030 unsigned char *sha;
2031 __be32 sip, tip;
2032
2033 if (!dev)
2034 return;
2035
2036 skb_reset_network_header(skb);
2037 arp = arp_hdr(skb);
2038
2039 if (arp->ar_op != htons(ARPOP_REQUEST))
2040 return;
2041
2042 arp_ptr = (unsigned char *)(arp + 1);
2043 sha = arp_ptr;
2044 arp_ptr += dev->addr_len;
2045 memcpy(&sip, arp_ptr, sizeof(sip));
2046 arp_ptr += sizeof(sip);
2047 arp_ptr += dev->addr_len;
2048 memcpy(&tip, arp_ptr, sizeof(tip));
2049
2050 if (tip != pi->iscsi_ipv4addr)
2051 return;
2052
2053 arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
2054 pi->iscsic.mac_addr, sha);
2055
2056}
2057
2058static inline int is_arp(struct sk_buff *skb)
2059{
2060 return skb->protocol == htons(ETH_P_ARP);
2061}
2062
2063static void cxgb3_process_iscsi_prov_pack(struct port_info *pi,
2064 struct sk_buff *skb)
2065{
2066 if (is_arp(skb)) {
2067 cxgb3_arp_process(pi, skb);
2068 return;
2069 }
2070
2071 if (pi->iscsic.recv)
2072 pi->iscsic.recv(pi, skb);
2073
2074}
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2089 struct sk_buff *skb, int pad, int lro)
2090{
2091 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2092 struct sge_qset *qs = rspq_to_qset(rq);
2093 struct port_info *pi;
2094
2095 skb_pull(skb, sizeof(*p) + pad);
2096 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2097 pi = netdev_priv(skb->dev);
2098 if ((skb->dev->features & NETIF_F_RXCSUM) && p->csum_valid &&
2099 p->csum == htons(0xffff) && !p->fragment) {
2100 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2101 skb->ip_summed = CHECKSUM_UNNECESSARY;
2102 } else
2103 skb_checksum_none_assert(skb);
2104 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2105
2106 if (p->vlan_valid) {
2107 qs->port_stats[SGE_PSTAT_VLANEX]++;
2108 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(p->vlan));
2109 }
2110 if (rq->polling) {
2111 if (lro)
2112 napi_gro_receive(&qs->napi, skb);
2113 else {
2114 if (unlikely(pi->iscsic.flags))
2115 cxgb3_process_iscsi_prov_pack(pi, skb);
2116 netif_receive_skb(skb);
2117 }
2118 } else
2119 netif_rx(skb);
2120}
2121
2122static inline int is_eth_tcp(u32 rss)
2123{
2124 return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2125}
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2139 struct sge_fl *fl, int len, int complete)
2140{
2141 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2142 struct port_info *pi = netdev_priv(qs->netdev);
2143 struct sk_buff *skb = NULL;
2144 struct cpl_rx_pkt *cpl;
2145 skb_frag_t *rx_frag;
2146 int nr_frags;
2147 int offset = 0;
2148
2149 if (!qs->nomem) {
2150 skb = napi_get_frags(&qs->napi);
2151 qs->nomem = !skb;
2152 }
2153
2154 fl->credits--;
2155
2156 pci_dma_sync_single_for_cpu(adap->pdev,
2157 dma_unmap_addr(sd, dma_addr),
2158 fl->buf_size - SGE_PG_RSVD,
2159 PCI_DMA_FROMDEVICE);
2160
2161 (*sd->pg_chunk.p_cnt)--;
2162 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
2163 pci_unmap_page(adap->pdev,
2164 sd->pg_chunk.mapping,
2165 fl->alloc_size,
2166 PCI_DMA_FROMDEVICE);
2167
2168 if (!skb) {
2169 put_page(sd->pg_chunk.page);
2170 if (complete)
2171 qs->nomem = 0;
2172 return;
2173 }
2174
2175 rx_frag = skb_shinfo(skb)->frags;
2176 nr_frags = skb_shinfo(skb)->nr_frags;
2177
2178 if (!nr_frags) {
2179 offset = 2 + sizeof(struct cpl_rx_pkt);
2180 cpl = qs->lro_va = sd->pg_chunk.va + 2;
2181
2182 if ((qs->netdev->features & NETIF_F_RXCSUM) &&
2183 cpl->csum_valid && cpl->csum == htons(0xffff)) {
2184 skb->ip_summed = CHECKSUM_UNNECESSARY;
2185 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2186 } else
2187 skb->ip_summed = CHECKSUM_NONE;
2188 } else
2189 cpl = qs->lro_va;
2190
2191 len -= offset;
2192
2193 rx_frag += nr_frags;
2194 __skb_frag_set_page(rx_frag, sd->pg_chunk.page);
2195 skb_frag_off_set(rx_frag, sd->pg_chunk.offset + offset);
2196 skb_frag_size_set(rx_frag, len);
2197
2198 skb->len += len;
2199 skb->data_len += len;
2200 skb->truesize += len;
2201 skb_shinfo(skb)->nr_frags++;
2202
2203 if (!complete)
2204 return;
2205
2206 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2207
2208 if (cpl->vlan_valid) {
2209 qs->port_stats[SGE_PSTAT_VLANEX]++;
2210 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cpl->vlan));
2211 }
2212 napi_gro_frags(&qs->napi);
2213}
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2225{
2226 unsigned int credits;
2227
2228#if USE_GTS
2229 if (flags & F_RSPD_TXQ0_GTS)
2230 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2231#endif
2232
2233 credits = G_RSPD_TXQ0_CR(flags);
2234 if (credits)
2235 qs->txq[TXQ_ETH].processed += credits;
2236
2237 credits = G_RSPD_TXQ2_CR(flags);
2238 if (credits)
2239 qs->txq[TXQ_CTRL].processed += credits;
2240
2241# if USE_GTS
2242 if (flags & F_RSPD_TXQ1_GTS)
2243 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2244# endif
2245 credits = G_RSPD_TXQ1_CR(flags);
2246 if (credits)
2247 qs->txq[TXQ_OFLD].processed += credits;
2248}
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2261 unsigned int sleeping)
2262{
2263 if (sleeping & F_RSPD_TXQ0_GTS) {
2264 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2265
2266 if (txq->cleaned + txq->in_use != txq->processed &&
2267 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2268 set_bit(TXQ_RUNNING, &txq->flags);
2269 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2270 V_EGRCNTX(txq->cntxt_id));
2271 }
2272 }
2273
2274 if (sleeping & F_RSPD_TXQ1_GTS) {
2275 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2276
2277 if (txq->cleaned + txq->in_use != txq->processed &&
2278 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2279 set_bit(TXQ_RUNNING, &txq->flags);
2280 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2281 V_EGRCNTX(txq->cntxt_id));
2282 }
2283 }
2284}
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294static inline int is_new_response(const struct rsp_desc *r,
2295 const struct sge_rspq *q)
2296{
2297 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2298}
2299
2300static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2301{
2302 q->pg_skb = NULL;
2303 q->rx_recycle_buf = 0;
2304}
2305
2306#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2307#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2308 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2309 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2310 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2311
2312
2313#define NOMEM_INTR_DELAY 2500
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330static int process_responses(struct adapter *adap, struct sge_qset *qs,
2331 int budget)
2332{
2333 struct sge_rspq *q = &qs->rspq;
2334 struct rsp_desc *r = &q->desc[q->cidx];
2335 int budget_left = budget;
2336 unsigned int sleeping = 0;
2337 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2338 int ngathered = 0;
2339
2340 q->next_holdoff = q->holdoff_tmr;
2341
2342 while (likely(budget_left && is_new_response(r, q))) {
2343 int packet_complete, eth, ethpad = 2;
2344 int lro = !!(qs->netdev->features & NETIF_F_GRO);
2345 struct sk_buff *skb = NULL;
2346 u32 len, flags;
2347 __be32 rss_hi, rss_lo;
2348
2349 dma_rmb();
2350 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2351 rss_hi = *(const __be32 *)r;
2352 rss_lo = r->rss_hdr.rss_hash_val;
2353 flags = ntohl(r->flags);
2354
2355 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2356 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2357 if (!skb)
2358 goto no_mem;
2359
2360 __skb_put_data(skb, r, AN_PKT_SIZE);
2361 skb->data[0] = CPL_ASYNC_NOTIF;
2362 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2363 q->async_notif++;
2364 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2365 skb = get_imm_packet(r);
2366 if (unlikely(!skb)) {
2367no_mem:
2368 q->next_holdoff = NOMEM_INTR_DELAY;
2369 q->nomem++;
2370
2371 budget_left--;
2372 break;
2373 }
2374 q->imm_data++;
2375 ethpad = 0;
2376 } else if ((len = ntohl(r->len_cq)) != 0) {
2377 struct sge_fl *fl;
2378
2379 lro &= eth && is_eth_tcp(rss_hi);
2380
2381 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2382 if (fl->use_pages) {
2383 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2384
2385 net_prefetch(addr);
2386 __refill_fl(adap, fl);
2387 if (lro > 0) {
2388 lro_add_page(adap, qs, fl,
2389 G_RSPD_LEN(len),
2390 flags & F_RSPD_EOP);
2391 goto next_fl;
2392 }
2393
2394 skb = get_packet_pg(adap, fl, q,
2395 G_RSPD_LEN(len),
2396 eth ?
2397 SGE_RX_DROP_THRES : 0);
2398 q->pg_skb = skb;
2399 } else
2400 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2401 eth ? SGE_RX_DROP_THRES : 0);
2402 if (unlikely(!skb)) {
2403 if (!eth)
2404 goto no_mem;
2405 q->rx_drops++;
2406 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2407 __skb_pull(skb, 2);
2408next_fl:
2409 if (++fl->cidx == fl->size)
2410 fl->cidx = 0;
2411 } else
2412 q->pure_rsps++;
2413
2414 if (flags & RSPD_CTRL_MASK) {
2415 sleeping |= flags & RSPD_GTS_MASK;
2416 handle_rsp_cntrl_info(qs, flags);
2417 }
2418
2419 r++;
2420 if (unlikely(++q->cidx == q->size)) {
2421 q->cidx = 0;
2422 q->gen ^= 1;
2423 r = q->desc;
2424 }
2425 prefetch(r);
2426
2427 if (++q->credits >= (q->size / 4)) {
2428 refill_rspq(adap, q, q->credits);
2429 q->credits = 0;
2430 }
2431
2432 packet_complete = flags &
2433 (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2434 F_RSPD_ASYNC_NOTIF);
2435
2436 if (skb != NULL && packet_complete) {
2437 if (eth)
2438 rx_eth(adap, q, skb, ethpad, lro);
2439 else {
2440 q->offload_pkts++;
2441
2442 skb->csum = rss_hi;
2443 skb->priority = rss_lo;
2444 ngathered = rx_offload(&adap->tdev, q, skb,
2445 offload_skbs,
2446 ngathered);
2447 }
2448
2449 if (flags & F_RSPD_EOP)
2450 clear_rspq_bufstate(q);
2451 }
2452 --budget_left;
2453 }
2454
2455 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2456
2457 if (sleeping)
2458 check_ring_db(adap, qs, sleeping);
2459
2460 smp_mb();
2461 if (unlikely(qs->txq_stopped != 0))
2462 restart_tx(qs);
2463
2464 budget -= budget_left;
2465 return budget;
2466}
2467
2468static inline int is_pure_response(const struct rsp_desc *r)
2469{
2470 __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2471
2472 return (n | r->len_cq) == 0;
2473}
2474
2475
2476
2477
2478
2479
2480
2481
2482static int napi_rx_handler(struct napi_struct *napi, int budget)
2483{
2484 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2485 struct adapter *adap = qs->adap;
2486 int work_done = process_responses(adap, qs, budget);
2487
2488 if (likely(work_done < budget)) {
2489 napi_complete_done(napi, work_done);
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2506 V_NEWTIMER(qs->rspq.next_holdoff) |
2507 V_NEWINDEX(qs->rspq.cidx));
2508 }
2509 return work_done;
2510}
2511
2512
2513
2514
2515static inline int napi_is_scheduled(struct napi_struct *napi)
2516{
2517 return test_bit(NAPI_STATE_SCHED, &napi->state);
2518}
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2535 struct rsp_desc *r)
2536{
2537 struct sge_rspq *q = &qs->rspq;
2538 unsigned int sleeping = 0;
2539
2540 do {
2541 u32 flags = ntohl(r->flags);
2542
2543 r++;
2544 if (unlikely(++q->cidx == q->size)) {
2545 q->cidx = 0;
2546 q->gen ^= 1;
2547 r = q->desc;
2548 }
2549 prefetch(r);
2550
2551 if (flags & RSPD_CTRL_MASK) {
2552 sleeping |= flags & RSPD_GTS_MASK;
2553 handle_rsp_cntrl_info(qs, flags);
2554 }
2555
2556 q->pure_rsps++;
2557 if (++q->credits >= (q->size / 4)) {
2558 refill_rspq(adap, q, q->credits);
2559 q->credits = 0;
2560 }
2561 if (!is_new_response(r, q))
2562 break;
2563 dma_rmb();
2564 } while (is_pure_response(r));
2565
2566 if (sleeping)
2567 check_ring_db(adap, qs, sleeping);
2568
2569 smp_mb();
2570 if (unlikely(qs->txq_stopped != 0))
2571 restart_tx(qs);
2572
2573 return is_new_response(r, q);
2574}
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2592{
2593 struct sge_qset *qs = rspq_to_qset(q);
2594 struct rsp_desc *r = &q->desc[q->cidx];
2595
2596 if (!is_new_response(r, q))
2597 return -1;
2598 dma_rmb();
2599 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2600 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2601 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2602 return 0;
2603 }
2604 napi_schedule(&qs->napi);
2605 return 1;
2606}
2607
2608
2609
2610
2611
2612static irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2613{
2614 struct sge_qset *qs = cookie;
2615 struct adapter *adap = qs->adap;
2616 struct sge_rspq *q = &qs->rspq;
2617
2618 spin_lock(&q->lock);
2619 if (process_responses(adap, qs, -1) == 0)
2620 q->unhandled_irqs++;
2621 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2622 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2623 spin_unlock(&q->lock);
2624 return IRQ_HANDLED;
2625}
2626
2627
2628
2629
2630
2631static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2632{
2633 struct sge_qset *qs = cookie;
2634 struct sge_rspq *q = &qs->rspq;
2635
2636 spin_lock(&q->lock);
2637
2638 if (handle_responses(qs->adap, q) < 0)
2639 q->unhandled_irqs++;
2640 spin_unlock(&q->lock);
2641 return IRQ_HANDLED;
2642}
2643
2644
2645
2646
2647
2648
2649
2650static irqreturn_t t3_intr_msi(int irq, void *cookie)
2651{
2652 int new_packets = 0;
2653 struct adapter *adap = cookie;
2654 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2655
2656 spin_lock(&q->lock);
2657
2658 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2659 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2660 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2661 new_packets = 1;
2662 }
2663
2664 if (adap->params.nports == 2 &&
2665 process_responses(adap, &adap->sge.qs[1], -1)) {
2666 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2667
2668 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2669 V_NEWTIMER(q1->next_holdoff) |
2670 V_NEWINDEX(q1->cidx));
2671 new_packets = 1;
2672 }
2673
2674 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2675 q->unhandled_irqs++;
2676
2677 spin_unlock(&q->lock);
2678 return IRQ_HANDLED;
2679}
2680
2681static int rspq_check_napi(struct sge_qset *qs)
2682{
2683 struct sge_rspq *q = &qs->rspq;
2684
2685 if (!napi_is_scheduled(&qs->napi) &&
2686 is_new_response(&q->desc[q->cidx], q)) {
2687 napi_schedule(&qs->napi);
2688 return 1;
2689 }
2690 return 0;
2691}
2692
2693
2694
2695
2696
2697
2698
2699
2700static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2701{
2702 int new_packets;
2703 struct adapter *adap = cookie;
2704 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2705
2706 spin_lock(&q->lock);
2707
2708 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2709 if (adap->params.nports == 2)
2710 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2711 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2712 q->unhandled_irqs++;
2713
2714 spin_unlock(&q->lock);
2715 return IRQ_HANDLED;
2716}
2717
2718
2719
2720
2721static inline int process_responses_gts(struct adapter *adap,
2722 struct sge_rspq *rq)
2723{
2724 int work;
2725
2726 work = process_responses(adap, rspq_to_qset(rq), -1);
2727 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2728 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2729 return work;
2730}
2731
2732
2733
2734
2735
2736
2737
2738static irqreturn_t t3_intr(int irq, void *cookie)
2739{
2740 int work_done, w0, w1;
2741 struct adapter *adap = cookie;
2742 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2743 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2744
2745 spin_lock(&q0->lock);
2746
2747 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2748 w1 = adap->params.nports == 2 &&
2749 is_new_response(&q1->desc[q1->cidx], q1);
2750
2751 if (likely(w0 | w1)) {
2752 t3_write_reg(adap, A_PL_CLI, 0);
2753 t3_read_reg(adap, A_PL_CLI);
2754
2755 if (likely(w0))
2756 process_responses_gts(adap, q0);
2757
2758 if (w1)
2759 process_responses_gts(adap, q1);
2760
2761 work_done = w0 | w1;
2762 } else
2763 work_done = t3_slow_intr_handler(adap);
2764
2765 spin_unlock(&q0->lock);
2766 return IRQ_RETVAL(work_done != 0);
2767}
2768
2769
2770
2771
2772
2773
2774
2775
2776static irqreturn_t t3b_intr(int irq, void *cookie)
2777{
2778 u32 map;
2779 struct adapter *adap = cookie;
2780 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2781
2782 t3_write_reg(adap, A_PL_CLI, 0);
2783 map = t3_read_reg(adap, A_SG_DATA_INTR);
2784
2785 if (unlikely(!map))
2786 return IRQ_NONE;
2787
2788 spin_lock(&q0->lock);
2789
2790 if (unlikely(map & F_ERRINTR))
2791 t3_slow_intr_handler(adap);
2792
2793 if (likely(map & 1))
2794 process_responses_gts(adap, q0);
2795
2796 if (map & 2)
2797 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2798
2799 spin_unlock(&q0->lock);
2800 return IRQ_HANDLED;
2801}
2802
2803
2804
2805
2806
2807
2808
2809
2810static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2811{
2812 u32 map;
2813 struct adapter *adap = cookie;
2814 struct sge_qset *qs0 = &adap->sge.qs[0];
2815 struct sge_rspq *q0 = &qs0->rspq;
2816
2817 t3_write_reg(adap, A_PL_CLI, 0);
2818 map = t3_read_reg(adap, A_SG_DATA_INTR);
2819
2820 if (unlikely(!map))
2821 return IRQ_NONE;
2822
2823 spin_lock(&q0->lock);
2824
2825 if (unlikely(map & F_ERRINTR))
2826 t3_slow_intr_handler(adap);
2827
2828 if (likely(map & 1))
2829 napi_schedule(&qs0->napi);
2830
2831 if (map & 2)
2832 napi_schedule(&adap->sge.qs[1].napi);
2833
2834 spin_unlock(&q0->lock);
2835 return IRQ_HANDLED;
2836}
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2848{
2849 if (adap->flags & USING_MSIX)
2850 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2851 if (adap->flags & USING_MSI)
2852 return polling ? t3_intr_msi_napi : t3_intr_msi;
2853 if (adap->params.rev > 0)
2854 return polling ? t3b_intr_napi : t3b_intr;
2855 return t3_intr;
2856}
2857
2858#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2859 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2860 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2861 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2862 F_HIRCQPARITYERROR)
2863#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2864#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2865 F_RSPQDISABLED)
2866
2867
2868
2869
2870
2871
2872
2873void t3_sge_err_intr_handler(struct adapter *adapter)
2874{
2875 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2876 ~F_FLEMPTY;
2877
2878 if (status & SGE_PARERR)
2879 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2880 status & SGE_PARERR);
2881 if (status & SGE_FRAMINGERR)
2882 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2883 status & SGE_FRAMINGERR);
2884
2885 if (status & F_RSPQCREDITOVERFOW)
2886 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2887
2888 if (status & F_RSPQDISABLED) {
2889 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2890
2891 CH_ALERT(adapter,
2892 "packet delivered to disabled response queue "
2893 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2894 }
2895
2896 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2897 queue_work(cxgb3_wq, &adapter->db_drop_task);
2898
2899 if (status & (F_HIPRIORITYDBFULL | F_LOPRIORITYDBFULL))
2900 queue_work(cxgb3_wq, &adapter->db_full_task);
2901
2902 if (status & (F_HIPRIORITYDBEMPTY | F_LOPRIORITYDBEMPTY))
2903 queue_work(cxgb3_wq, &adapter->db_empty_task);
2904
2905 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2906 if (status & SGE_FATALERR)
2907 t3_fatal_err(adapter);
2908}
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928static void sge_timer_tx(struct timer_list *t)
2929{
2930 struct sge_qset *qs = from_timer(qs, t, tx_reclaim_timer);
2931 struct port_info *pi = netdev_priv(qs->netdev);
2932 struct adapter *adap = pi->adapter;
2933 unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2934 unsigned long next_period;
2935
2936 if (__netif_tx_trylock(qs->tx_q)) {
2937 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2938 TX_RECLAIM_TIMER_CHUNK);
2939 __netif_tx_unlock(qs->tx_q);
2940 }
2941
2942 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2943 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2944 TX_RECLAIM_TIMER_CHUNK);
2945 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2946 }
2947
2948 next_period = TX_RECLAIM_PERIOD >>
2949 (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2950 TX_RECLAIM_TIMER_CHUNK);
2951 mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2952}
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968static void sge_timer_rx(struct timer_list *t)
2969{
2970 spinlock_t *lock;
2971 struct sge_qset *qs = from_timer(qs, t, rx_reclaim_timer);
2972 struct port_info *pi = netdev_priv(qs->netdev);
2973 struct adapter *adap = pi->adapter;
2974 u32 status;
2975
2976 lock = adap->params.rev > 0 ?
2977 &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2978
2979 if (!spin_trylock_irq(lock))
2980 goto out;
2981
2982 if (napi_is_scheduled(&qs->napi))
2983 goto unlock;
2984
2985 if (adap->params.rev < 4) {
2986 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2987
2988 if (status & (1 << qs->rspq.cntxt_id)) {
2989 qs->rspq.starved++;
2990 if (qs->rspq.credits) {
2991 qs->rspq.credits--;
2992 refill_rspq(adap, &qs->rspq, 1);
2993 qs->rspq.restarted++;
2994 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2995 1 << qs->rspq.cntxt_id);
2996 }
2997 }
2998 }
2999
3000 if (qs->fl[0].credits < qs->fl[0].size)
3001 __refill_fl(adap, &qs->fl[0]);
3002 if (qs->fl[1].credits < qs->fl[1].size)
3003 __refill_fl(adap, &qs->fl[1]);
3004
3005unlock:
3006 spin_unlock_irq(lock);
3007out:
3008 mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3009}
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
3020{
3021 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);
3022 qs->rspq.polling = p->polling;
3023 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
3024}
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
3043 int irq_vec_idx, const struct qset_params *p,
3044 int ntxq, struct net_device *dev,
3045 struct netdev_queue *netdevq)
3046{
3047 int i, avail, ret = -ENOMEM;
3048 struct sge_qset *q = &adapter->sge.qs[id];
3049
3050 init_qset_cntxt(q, id);
3051 timer_setup(&q->tx_reclaim_timer, sge_timer_tx, 0);
3052 timer_setup(&q->rx_reclaim_timer, sge_timer_rx, 0);
3053
3054 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
3055 sizeof(struct rx_desc),
3056 sizeof(struct rx_sw_desc),
3057 &q->fl[0].phys_addr, &q->fl[0].sdesc);
3058 if (!q->fl[0].desc)
3059 goto err;
3060
3061 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
3062 sizeof(struct rx_desc),
3063 sizeof(struct rx_sw_desc),
3064 &q->fl[1].phys_addr, &q->fl[1].sdesc);
3065 if (!q->fl[1].desc)
3066 goto err;
3067
3068 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
3069 sizeof(struct rsp_desc), 0,
3070 &q->rspq.phys_addr, NULL);
3071 if (!q->rspq.desc)
3072 goto err;
3073
3074 for (i = 0; i < ntxq; ++i) {
3075
3076
3077
3078
3079 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3080
3081 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3082 sizeof(struct tx_desc), sz,
3083 &q->txq[i].phys_addr,
3084 &q->txq[i].sdesc);
3085 if (!q->txq[i].desc)
3086 goto err;
3087
3088 q->txq[i].gen = 1;
3089 q->txq[i].size = p->txq_size[i];
3090 spin_lock_init(&q->txq[i].lock);
3091 skb_queue_head_init(&q->txq[i].sendq);
3092 }
3093
3094 INIT_WORK(&q->txq[TXQ_OFLD].qresume_task, restart_offloadq);
3095 INIT_WORK(&q->txq[TXQ_CTRL].qresume_task, restart_ctrlq);
3096
3097 q->fl[0].gen = q->fl[1].gen = 1;
3098 q->fl[0].size = p->fl_size;
3099 q->fl[1].size = p->jumbo_size;
3100
3101 q->rspq.gen = 1;
3102 q->rspq.size = p->rspq_size;
3103 spin_lock_init(&q->rspq.lock);
3104 skb_queue_head_init(&q->rspq.rx_queue);
3105
3106 q->txq[TXQ_ETH].stop_thres = nports *
3107 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3108
3109#if FL0_PG_CHUNK_SIZE > 0
3110 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3111#else
3112 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3113#endif
3114#if FL1_PG_CHUNK_SIZE > 0
3115 q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3116#else
3117 q->fl[1].buf_size = is_offload(adapter) ?
3118 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3119 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3120#endif
3121
3122 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3123 q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3124 q->fl[0].order = FL0_PG_ORDER;
3125 q->fl[1].order = FL1_PG_ORDER;
3126 q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3127 q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3128
3129 spin_lock_irq(&adapter->sge.reg_lock);
3130
3131
3132 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3133 q->rspq.phys_addr, q->rspq.size,
3134 q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3135 if (ret)
3136 goto err_unlock;
3137
3138 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3139 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3140 q->fl[i].phys_addr, q->fl[i].size,
3141 q->fl[i].buf_size - SGE_PG_RSVD,
3142 p->cong_thres, 1, 0);
3143 if (ret)
3144 goto err_unlock;
3145 }
3146
3147 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3148 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3149 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3150 1, 0);
3151 if (ret)
3152 goto err_unlock;
3153
3154 if (ntxq > 1) {
3155 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3156 USE_GTS, SGE_CNTXT_OFLD, id,
3157 q->txq[TXQ_OFLD].phys_addr,
3158 q->txq[TXQ_OFLD].size, 0, 1, 0);
3159 if (ret)
3160 goto err_unlock;
3161 }
3162
3163 if (ntxq > 2) {
3164 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3165 SGE_CNTXT_CTRL, id,
3166 q->txq[TXQ_CTRL].phys_addr,
3167 q->txq[TXQ_CTRL].size,
3168 q->txq[TXQ_CTRL].token, 1, 0);
3169 if (ret)
3170 goto err_unlock;
3171 }
3172
3173 spin_unlock_irq(&adapter->sge.reg_lock);
3174
3175 q->adap = adapter;
3176 q->netdev = dev;
3177 q->tx_q = netdevq;
3178 t3_update_qset_coalesce(q, p);
3179
3180 avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3181 GFP_KERNEL | __GFP_COMP);
3182 if (!avail) {
3183 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3184 ret = -ENOMEM;
3185 goto err;
3186 }
3187 if (avail < q->fl[0].size)
3188 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3189 avail);
3190
3191 avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3192 GFP_KERNEL | __GFP_COMP);
3193 if (avail < q->fl[1].size)
3194 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3195 avail);
3196 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3197
3198 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3199 V_NEWTIMER(q->rspq.holdoff_tmr));
3200
3201 return 0;
3202
3203err_unlock:
3204 spin_unlock_irq(&adapter->sge.reg_lock);
3205err:
3206 t3_free_qset(adapter, q);
3207 return ret;
3208}
3209
3210
3211
3212
3213
3214
3215
3216void t3_start_sge_timers(struct adapter *adap)
3217{
3218 int i;
3219
3220 for (i = 0; i < SGE_QSETS; ++i) {
3221 struct sge_qset *q = &adap->sge.qs[i];
3222
3223 if (q->tx_reclaim_timer.function)
3224 mod_timer(&q->tx_reclaim_timer,
3225 jiffies + TX_RECLAIM_PERIOD);
3226
3227 if (q->rx_reclaim_timer.function)
3228 mod_timer(&q->rx_reclaim_timer,
3229 jiffies + RX_RECLAIM_PERIOD);
3230 }
3231}
3232
3233
3234
3235
3236
3237
3238
3239void t3_stop_sge_timers(struct adapter *adap)
3240{
3241 int i;
3242
3243 for (i = 0; i < SGE_QSETS; ++i) {
3244 struct sge_qset *q = &adap->sge.qs[i];
3245
3246 if (q->tx_reclaim_timer.function)
3247 del_timer_sync(&q->tx_reclaim_timer);
3248 if (q->rx_reclaim_timer.function)
3249 del_timer_sync(&q->rx_reclaim_timer);
3250 }
3251}
3252
3253
3254
3255
3256
3257
3258
3259void t3_free_sge_resources(struct adapter *adap)
3260{
3261 int i;
3262
3263 for (i = 0; i < SGE_QSETS; ++i)
3264 t3_free_qset(adap, &adap->sge.qs[i]);
3265}
3266
3267
3268
3269
3270
3271
3272
3273
3274void t3_sge_start(struct adapter *adap)
3275{
3276 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3277}
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291void t3_sge_stop_dma(struct adapter *adap)
3292{
3293 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3294}
3295
3296
3297
3298
3299
3300
3301
3302
3303void t3_sge_stop(struct adapter *adap)
3304{
3305 int i;
3306
3307 t3_sge_stop_dma(adap);
3308
3309 for (i = 0; i < SGE_QSETS; ++i) {
3310 struct sge_qset *qs = &adap->sge.qs[i];
3311
3312 cancel_work_sync(&qs->txq[TXQ_OFLD].qresume_task);
3313 cancel_work_sync(&qs->txq[TXQ_CTRL].qresume_task);
3314 }
3315}
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327void t3_sge_init(struct adapter *adap, struct sge_params *p)
3328{
3329 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3330
3331 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3332 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3333 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3334 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3335#if SGE_NUM_GENBITS == 1
3336 ctrl |= F_EGRGENCTRL;
3337#endif
3338 if (adap->params.rev > 0) {
3339 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3340 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3341 }
3342 t3_write_reg(adap, A_SG_CONTROL, ctrl);
3343 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3344 V_LORCQDRBTHRSH(512));
3345 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3346 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3347 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3348 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3349 adap->params.rev < T3_REV_C ? 1000 : 500);
3350 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3351 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3352 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3353 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3354 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3355}
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3367{
3368 int i;
3369
3370 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3371 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3372
3373 for (i = 0; i < SGE_QSETS; ++i) {
3374 struct qset_params *q = p->qset + i;
3375
3376 q->polling = adap->params.rev > 0;
3377 q->coalesce_usecs = 5;
3378 q->rspq_size = 1024;
3379 q->fl_size = 1024;
3380 q->jumbo_size = 512;
3381 q->txq_size[TXQ_ETH] = 1024;
3382 q->txq_size[TXQ_OFLD] = 1024;
3383 q->txq_size[TXQ_CTRL] = 256;
3384 q->cong_thres = 0;
3385 }
3386
3387 spin_lock_init(&adap->sge.reg_lock);
3388}
3389