1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/skbuff.h>
33#include <linux/netdevice.h>
34#include <linux/etherdevice.h>
35#include <linux/if_vlan.h>
36#include <linux/ip.h>
37#include <linux/tcp.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/prefetch.h>
41#include <net/arp.h>
42#include "common.h"
43#include "regs.h"
44#include "sge_defs.h"
45#include "t3_cpl.h"
46#include "firmware_exports.h"
47#include "cxgb3_offload.h"
48
49#define USE_GTS 0
50
51#define SGE_RX_SM_BUF_SIZE 1536
52
53#define SGE_RX_COPY_THRES 256
54#define SGE_RX_PULL_LEN 128
55
56#define SGE_PG_RSVD SMP_CACHE_BYTES
57
58
59
60
61
62#define FL0_PG_CHUNK_SIZE 2048
63#define FL0_PG_ORDER 0
64#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
65#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
66#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
67#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
68
69#define SGE_RX_DROP_THRES 16
70#define RX_RECLAIM_PERIOD (HZ/4)
71
72
73
74
75#define MAX_RX_REFILL 16U
76
77
78
79
80#define TX_RECLAIM_PERIOD (HZ / 4)
81#define TX_RECLAIM_TIMER_CHUNK 64U
82#define TX_RECLAIM_CHUNK 16U
83
84
85#define WR_LEN (WR_FLITS * 8)
86
87
88
89
90enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
91
92
93enum {
94 TXQ_RUNNING = 1 << 0,
95 TXQ_LAST_PKT_DB = 1 << 1,
96};
97
98struct tx_desc {
99 __be64 flit[TX_DESC_FLITS];
100};
101
102struct rx_desc {
103 __be32 addr_lo;
104 __be32 len_gen;
105 __be32 gen2;
106 __be32 addr_hi;
107};
108
109struct tx_sw_desc {
110 struct sk_buff *skb;
111 u8 eop;
112 u8 addr_idx;
113 u8 fragidx;
114 s8 sflit;
115};
116
117struct rx_sw_desc {
118 union {
119 struct sk_buff *skb;
120 struct fl_pg_chunk pg_chunk;
121 };
122 DEFINE_DMA_UNMAP_ADDR(dma_addr);
123};
124
125struct rsp_desc {
126 struct rss_header rss_hdr;
127 __be32 flags;
128 __be32 len_cq;
129 u8 imm_data[47];
130 u8 intr_gen;
131};
132
133
134
135
136
137struct deferred_unmap_info {
138 struct pci_dev *pdev;
139 dma_addr_t addr[MAX_SKB_FRAGS + 1];
140};
141
142
143
144
145
146
147
148
149
150static u8 flit_desc_map[] = {
151 0,
152#if SGE_NUM_GENBITS == 1
153 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
154 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
155 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
156 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
157#elif SGE_NUM_GENBITS == 2
158 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
159 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
160 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
161 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
162#else
163# error "SGE_NUM_GENBITS must be 1 or 2"
164#endif
165};
166
167static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
168{
169 return container_of(q, struct sge_qset, fl[qidx]);
170}
171
172static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
173{
174 return container_of(q, struct sge_qset, rspq);
175}
176
177static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
178{
179 return container_of(q, struct sge_qset, txq[qidx]);
180}
181
182
183
184
185
186
187
188
189
190
191static inline void refill_rspq(struct adapter *adapter,
192 const struct sge_rspq *q, unsigned int credits)
193{
194 rmb();
195 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
196 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
197}
198
199
200
201
202
203
204
205static inline int need_skb_unmap(void)
206{
207#ifdef CONFIG_NEED_DMA_MAP_STATE
208 return 1;
209#else
210 return 0;
211#endif
212}
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
237 unsigned int cidx, struct pci_dev *pdev)
238{
239 const struct sg_ent *sgp;
240 struct tx_sw_desc *d = &q->sdesc[cidx];
241 int nfrags, frag_idx, curflit, j = d->addr_idx;
242
243 sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
244 frag_idx = d->fragidx;
245
246 if (frag_idx == 0 && skb_headlen(skb)) {
247 dma_unmap_single(&pdev->dev, be64_to_cpu(sgp->addr[0]),
248 skb_headlen(skb), DMA_TO_DEVICE);
249 j = 1;
250 }
251
252 curflit = d->sflit + 1 + j;
253 nfrags = skb_shinfo(skb)->nr_frags;
254
255 while (frag_idx < nfrags && curflit < WR_FLITS) {
256 dma_unmap_page(&pdev->dev, be64_to_cpu(sgp->addr[j]),
257 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]),
258 DMA_TO_DEVICE);
259 j ^= 1;
260 if (j == 0) {
261 sgp++;
262 curflit++;
263 }
264 curflit++;
265 frag_idx++;
266 }
267
268 if (frag_idx < nfrags) {
269 d = cidx + 1 == q->size ? q->sdesc : d + 1;
270 d->fragidx = frag_idx;
271 d->addr_idx = j;
272 d->sflit = curflit - WR_FLITS - j;
273 }
274}
275
276
277
278
279
280
281
282
283
284
285static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
286 unsigned int n)
287{
288 struct tx_sw_desc *d;
289 struct pci_dev *pdev = adapter->pdev;
290 unsigned int cidx = q->cidx;
291
292 const int need_unmap = need_skb_unmap() &&
293 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
294
295 d = &q->sdesc[cidx];
296 while (n--) {
297 if (d->skb) {
298 if (need_unmap)
299 unmap_skb(d->skb, q, cidx, pdev);
300 if (d->eop) {
301 dev_consume_skb_any(d->skb);
302 d->skb = NULL;
303 }
304 }
305 ++d;
306 if (++cidx == q->size) {
307 cidx = 0;
308 d = q->sdesc;
309 }
310 }
311 q->cidx = cidx;
312}
313
314
315
316
317
318
319
320
321
322
323
324static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
325 struct sge_txq *q,
326 unsigned int chunk)
327{
328 unsigned int reclaim = q->processed - q->cleaned;
329
330 reclaim = min(chunk, reclaim);
331 if (reclaim) {
332 free_tx_desc(adapter, q, reclaim);
333 q->cleaned += reclaim;
334 q->in_use -= reclaim;
335 }
336 return q->processed - q->cleaned;
337}
338
339
340
341
342
343
344
345static inline int should_restart_tx(const struct sge_txq *q)
346{
347 unsigned int r = q->processed - q->cleaned;
348
349 return q->in_use - r < (q->size >> 1);
350}
351
352static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
353 struct rx_sw_desc *d)
354{
355 if (q->use_pages && d->pg_chunk.page) {
356 (*d->pg_chunk.p_cnt)--;
357 if (!*d->pg_chunk.p_cnt)
358 dma_unmap_page(&pdev->dev, d->pg_chunk.mapping,
359 q->alloc_size, DMA_FROM_DEVICE);
360
361 put_page(d->pg_chunk.page);
362 d->pg_chunk.page = NULL;
363 } else {
364 dma_unmap_single(&pdev->dev, dma_unmap_addr(d, dma_addr),
365 q->buf_size, DMA_FROM_DEVICE);
366 kfree_skb(d->skb);
367 d->skb = NULL;
368 }
369}
370
371
372
373
374
375
376
377
378
379static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
380{
381 unsigned int cidx = q->cidx;
382
383 while (q->credits--) {
384 struct rx_sw_desc *d = &q->sdesc[cidx];
385
386
387 clear_rx_desc(pdev, q, d);
388 if (++cidx == q->size)
389 cidx = 0;
390 }
391
392 if (q->pg_chunk.page) {
393 __free_pages(q->pg_chunk.page, q->order);
394 q->pg_chunk.page = NULL;
395 }
396}
397
398
399
400
401
402
403
404
405
406
407
408
409
410static inline int add_one_rx_buf(void *va, unsigned int len,
411 struct rx_desc *d, struct rx_sw_desc *sd,
412 unsigned int gen, struct pci_dev *pdev)
413{
414 dma_addr_t mapping;
415
416 mapping = dma_map_single(&pdev->dev, va, len, DMA_FROM_DEVICE);
417 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
418 return -ENOMEM;
419
420 dma_unmap_addr_set(sd, dma_addr, mapping);
421
422 d->addr_lo = cpu_to_be32(mapping);
423 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
424 dma_wmb();
425 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
426 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
427 return 0;
428}
429
430static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
431 unsigned int gen)
432{
433 d->addr_lo = cpu_to_be32(mapping);
434 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
435 dma_wmb();
436 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
437 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
438 return 0;
439}
440
441static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
442 struct rx_sw_desc *sd, gfp_t gfp,
443 unsigned int order)
444{
445 if (!q->pg_chunk.page) {
446 dma_addr_t mapping;
447
448 q->pg_chunk.page = alloc_pages(gfp, order);
449 if (unlikely(!q->pg_chunk.page))
450 return -ENOMEM;
451 q->pg_chunk.va = page_address(q->pg_chunk.page);
452 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
453 SGE_PG_RSVD;
454 q->pg_chunk.offset = 0;
455 mapping = dma_map_page(&adapter->pdev->dev, q->pg_chunk.page,
456 0, q->alloc_size, DMA_FROM_DEVICE);
457 if (unlikely(dma_mapping_error(&adapter->pdev->dev, mapping))) {
458 __free_pages(q->pg_chunk.page, order);
459 q->pg_chunk.page = NULL;
460 return -EIO;
461 }
462 q->pg_chunk.mapping = mapping;
463 }
464 sd->pg_chunk = q->pg_chunk;
465
466 prefetch(sd->pg_chunk.p_cnt);
467
468 q->pg_chunk.offset += q->buf_size;
469 if (q->pg_chunk.offset == (PAGE_SIZE << order))
470 q->pg_chunk.page = NULL;
471 else {
472 q->pg_chunk.va += q->buf_size;
473 get_page(q->pg_chunk.page);
474 }
475
476 if (sd->pg_chunk.offset == 0)
477 *sd->pg_chunk.p_cnt = 1;
478 else
479 *sd->pg_chunk.p_cnt += 1;
480
481 return 0;
482}
483
484static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
485{
486 if (q->pend_cred >= q->credits / 4) {
487 q->pend_cred = 0;
488 wmb();
489 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
490 }
491}
492
493
494
495
496
497
498
499
500
501
502
503
504static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
505{
506 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
507 struct rx_desc *d = &q->desc[q->pidx];
508 unsigned int count = 0;
509
510 while (n--) {
511 dma_addr_t mapping;
512 int err;
513
514 if (q->use_pages) {
515 if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
516 q->order))) {
517nomem: q->alloc_failed++;
518 break;
519 }
520 mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
521 dma_unmap_addr_set(sd, dma_addr, mapping);
522
523 add_one_rx_chunk(mapping, d, q->gen);
524 dma_sync_single_for_device(&adap->pdev->dev, mapping,
525 q->buf_size - SGE_PG_RSVD,
526 DMA_FROM_DEVICE);
527 } else {
528 void *buf_start;
529
530 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
531 if (!skb)
532 goto nomem;
533
534 sd->skb = skb;
535 buf_start = skb->data;
536 err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
537 q->gen, adap->pdev);
538 if (unlikely(err)) {
539 clear_rx_desc(adap->pdev, q, sd);
540 break;
541 }
542 }
543
544 d++;
545 sd++;
546 if (++q->pidx == q->size) {
547 q->pidx = 0;
548 q->gen ^= 1;
549 sd = q->sdesc;
550 d = q->desc;
551 }
552 count++;
553 }
554
555 q->credits += count;
556 q->pend_cred += count;
557 ring_fl_db(adap, q);
558
559 return count;
560}
561
562static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
563{
564 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
565 GFP_ATOMIC | __GFP_COMP);
566}
567
568
569
570
571
572
573
574
575
576
577static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
578 unsigned int idx)
579{
580 struct rx_desc *from = &q->desc[idx];
581 struct rx_desc *to = &q->desc[q->pidx];
582
583 q->sdesc[q->pidx] = q->sdesc[idx];
584 to->addr_lo = from->addr_lo;
585 to->addr_hi = from->addr_hi;
586 dma_wmb();
587 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
588 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
589
590 if (++q->pidx == q->size) {
591 q->pidx = 0;
592 q->gen ^= 1;
593 }
594
595 q->credits++;
596 q->pend_cred++;
597 ring_fl_db(adap, q);
598}
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
618 size_t sw_size, dma_addr_t * phys, void *metadata)
619{
620 size_t len = nelem * elem_size;
621 void *s = NULL;
622 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
623
624 if (!p)
625 return NULL;
626 if (sw_size && metadata) {
627 s = kcalloc(nelem, sw_size, GFP_KERNEL);
628
629 if (!s) {
630 dma_free_coherent(&pdev->dev, len, p, *phys);
631 return NULL;
632 }
633 *(void **)metadata = s;
634 }
635 return p;
636}
637
638
639
640
641
642
643
644
645
646static void t3_reset_qset(struct sge_qset *q)
647{
648 if (q->adap &&
649 !(q->adap->flags & NAPI_INIT)) {
650 memset(q, 0, sizeof(*q));
651 return;
652 }
653
654 q->adap = NULL;
655 memset(&q->rspq, 0, sizeof(q->rspq));
656 memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
657 memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
658 q->txq_stopped = 0;
659 q->tx_reclaim_timer.function = NULL;
660 q->rx_reclaim_timer.function = NULL;
661 q->nomem = 0;
662 napi_free_frags(&q->napi);
663}
664
665
666
667
668
669
670
671
672
673
674
675static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
676{
677 int i;
678 struct pci_dev *pdev = adapter->pdev;
679
680 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
681 if (q->fl[i].desc) {
682 spin_lock_irq(&adapter->sge.reg_lock);
683 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
684 spin_unlock_irq(&adapter->sge.reg_lock);
685 free_rx_bufs(pdev, &q->fl[i]);
686 kfree(q->fl[i].sdesc);
687 dma_free_coherent(&pdev->dev,
688 q->fl[i].size *
689 sizeof(struct rx_desc), q->fl[i].desc,
690 q->fl[i].phys_addr);
691 }
692
693 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
694 if (q->txq[i].desc) {
695 spin_lock_irq(&adapter->sge.reg_lock);
696 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
697 spin_unlock_irq(&adapter->sge.reg_lock);
698 if (q->txq[i].sdesc) {
699 free_tx_desc(adapter, &q->txq[i],
700 q->txq[i].in_use);
701 kfree(q->txq[i].sdesc);
702 }
703 dma_free_coherent(&pdev->dev,
704 q->txq[i].size *
705 sizeof(struct tx_desc),
706 q->txq[i].desc, q->txq[i].phys_addr);
707 __skb_queue_purge(&q->txq[i].sendq);
708 }
709
710 if (q->rspq.desc) {
711 spin_lock_irq(&adapter->sge.reg_lock);
712 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
713 spin_unlock_irq(&adapter->sge.reg_lock);
714 dma_free_coherent(&pdev->dev,
715 q->rspq.size * sizeof(struct rsp_desc),
716 q->rspq.desc, q->rspq.phys_addr);
717 }
718
719 t3_reset_qset(q);
720}
721
722
723
724
725
726
727
728
729static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
730{
731 qs->rspq.cntxt_id = id;
732 qs->fl[0].cntxt_id = 2 * id;
733 qs->fl[1].cntxt_id = 2 * id + 1;
734 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
735 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
736 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
737 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
738 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
739}
740
741
742
743
744
745
746
747
748static inline unsigned int sgl_len(unsigned int n)
749{
750
751 return (3 * n) / 2 + (n & 1);
752}
753
754
755
756
757
758
759
760
761static inline unsigned int flits_to_desc(unsigned int n)
762{
763 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
764 return flit_desc_map[n];
765}
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
783 unsigned int len, unsigned int drop_thres)
784{
785 struct sk_buff *skb = NULL;
786 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
787
788 prefetch(sd->skb->data);
789 fl->credits--;
790
791 if (len <= SGE_RX_COPY_THRES) {
792 skb = alloc_skb(len, GFP_ATOMIC);
793 if (likely(skb != NULL)) {
794 __skb_put(skb, len);
795 dma_sync_single_for_cpu(&adap->pdev->dev,
796 dma_unmap_addr(sd, dma_addr),
797 len, DMA_FROM_DEVICE);
798 memcpy(skb->data, sd->skb->data, len);
799 dma_sync_single_for_device(&adap->pdev->dev,
800 dma_unmap_addr(sd, dma_addr),
801 len, DMA_FROM_DEVICE);
802 } else if (!drop_thres)
803 goto use_orig_buf;
804recycle:
805 recycle_rx_buf(adap, fl, fl->cidx);
806 return skb;
807 }
808
809 if (unlikely(fl->credits < drop_thres) &&
810 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
811 GFP_ATOMIC | __GFP_COMP) == 0)
812 goto recycle;
813
814use_orig_buf:
815 dma_unmap_single(&adap->pdev->dev, dma_unmap_addr(sd, dma_addr),
816 fl->buf_size, DMA_FROM_DEVICE);
817 skb = sd->skb;
818 skb_put(skb, len);
819 __refill_fl(adap, fl);
820 return skb;
821}
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
843 struct sge_rspq *q, unsigned int len,
844 unsigned int drop_thres)
845{
846 struct sk_buff *newskb, *skb;
847 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
848
849 dma_addr_t dma_addr = dma_unmap_addr(sd, dma_addr);
850
851 newskb = skb = q->pg_skb;
852 if (!skb && (len <= SGE_RX_COPY_THRES)) {
853 newskb = alloc_skb(len, GFP_ATOMIC);
854 if (likely(newskb != NULL)) {
855 __skb_put(newskb, len);
856 dma_sync_single_for_cpu(&adap->pdev->dev, dma_addr,
857 len, DMA_FROM_DEVICE);
858 memcpy(newskb->data, sd->pg_chunk.va, len);
859 dma_sync_single_for_device(&adap->pdev->dev, dma_addr,
860 len, DMA_FROM_DEVICE);
861 } else if (!drop_thres)
862 return NULL;
863recycle:
864 fl->credits--;
865 recycle_rx_buf(adap, fl, fl->cidx);
866 q->rx_recycle_buf++;
867 return newskb;
868 }
869
870 if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
871 goto recycle;
872
873 prefetch(sd->pg_chunk.p_cnt);
874
875 if (!skb)
876 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
877
878 if (unlikely(!newskb)) {
879 if (!drop_thres)
880 return NULL;
881 goto recycle;
882 }
883
884 dma_sync_single_for_cpu(&adap->pdev->dev, dma_addr, len,
885 DMA_FROM_DEVICE);
886 (*sd->pg_chunk.p_cnt)--;
887 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
888 dma_unmap_page(&adap->pdev->dev, sd->pg_chunk.mapping,
889 fl->alloc_size, DMA_FROM_DEVICE);
890 if (!skb) {
891 __skb_put(newskb, SGE_RX_PULL_LEN);
892 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
893 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
894 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
895 len - SGE_RX_PULL_LEN);
896 newskb->len = len;
897 newskb->data_len = len - SGE_RX_PULL_LEN;
898 newskb->truesize += newskb->data_len;
899 } else {
900 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
901 sd->pg_chunk.page,
902 sd->pg_chunk.offset, len);
903 newskb->len += len;
904 newskb->data_len += len;
905 newskb->truesize += len;
906 }
907
908 fl->credits--;
909
910
911
912
913 return newskb;
914}
915
916
917
918
919
920
921
922static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
923{
924 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
925
926 if (skb) {
927 __skb_put(skb, IMMED_PKT_SIZE);
928 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
929 }
930 return skb;
931}
932
933
934
935
936
937
938
939
940static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
941{
942 unsigned int flits;
943
944 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
945 return 1;
946
947 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
948 if (skb_shinfo(skb)->gso_size)
949 flits++;
950 return flits_to_desc(flits);
951}
952
953
954
955
956
957
958
959
960static int map_skb(struct pci_dev *pdev, const struct sk_buff *skb,
961 dma_addr_t *addr)
962{
963 const skb_frag_t *fp, *end;
964 const struct skb_shared_info *si;
965
966 if (skb_headlen(skb)) {
967 *addr = dma_map_single(&pdev->dev, skb->data,
968 skb_headlen(skb), DMA_TO_DEVICE);
969 if (dma_mapping_error(&pdev->dev, *addr))
970 goto out_err;
971 addr++;
972 }
973
974 si = skb_shinfo(skb);
975 end = &si->frags[si->nr_frags];
976
977 for (fp = si->frags; fp < end; fp++) {
978 *addr = skb_frag_dma_map(&pdev->dev, fp, 0, skb_frag_size(fp),
979 DMA_TO_DEVICE);
980 if (dma_mapping_error(&pdev->dev, *addr))
981 goto unwind;
982 addr++;
983 }
984 return 0;
985
986unwind:
987 while (fp-- > si->frags)
988 dma_unmap_page(&pdev->dev, *--addr, skb_frag_size(fp),
989 DMA_TO_DEVICE);
990
991 dma_unmap_single(&pdev->dev, addr[-1], skb_headlen(skb),
992 DMA_TO_DEVICE);
993out_err:
994 return -ENOMEM;
995}
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009static inline unsigned int write_sgl(const struct sk_buff *skb,
1010 struct sg_ent *sgp, unsigned char *start,
1011 unsigned int len, const dma_addr_t *addr)
1012{
1013 unsigned int i, j = 0, k = 0, nfrags;
1014
1015 if (len) {
1016 sgp->len[0] = cpu_to_be32(len);
1017 sgp->addr[j++] = cpu_to_be64(addr[k++]);
1018 }
1019
1020 nfrags = skb_shinfo(skb)->nr_frags;
1021 for (i = 0; i < nfrags; i++) {
1022 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1023
1024 sgp->len[j] = cpu_to_be32(skb_frag_size(frag));
1025 sgp->addr[j] = cpu_to_be64(addr[k++]);
1026 j ^= 1;
1027 if (j == 0)
1028 ++sgp;
1029 }
1030 if (j)
1031 sgp->len[j] = 0;
1032 return ((nfrags + (len != 0)) * 3) / 2 + j;
1033}
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1048{
1049#if USE_GTS
1050 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1051 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1052 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1053 t3_write_reg(adap, A_SG_KDOORBELL,
1054 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1055 }
1056#else
1057 wmb();
1058 t3_write_reg(adap, A_SG_KDOORBELL,
1059 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1060#endif
1061}
1062
1063static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1064{
1065#if SGE_NUM_GENBITS == 2
1066 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1067#endif
1068}
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1090 struct tx_desc *d, unsigned int pidx,
1091 const struct sge_txq *q,
1092 const struct sg_ent *sgl,
1093 unsigned int flits, unsigned int sgl_flits,
1094 unsigned int gen, __be32 wr_hi,
1095 __be32 wr_lo)
1096{
1097 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1098 struct tx_sw_desc *sd = &q->sdesc[pidx];
1099
1100 sd->skb = skb;
1101 if (need_skb_unmap()) {
1102 sd->fragidx = 0;
1103 sd->addr_idx = 0;
1104 sd->sflit = flits;
1105 }
1106
1107 if (likely(ndesc == 1)) {
1108 sd->eop = 1;
1109 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1110 V_WR_SGLSFLT(flits)) | wr_hi;
1111 dma_wmb();
1112 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1113 V_WR_GEN(gen)) | wr_lo;
1114 wr_gen2(d, gen);
1115 } else {
1116 unsigned int ogen = gen;
1117 const u64 *fp = (const u64 *)sgl;
1118 struct work_request_hdr *wp = wrp;
1119
1120 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1121 V_WR_SGLSFLT(flits)) | wr_hi;
1122
1123 while (sgl_flits) {
1124 unsigned int avail = WR_FLITS - flits;
1125
1126 if (avail > sgl_flits)
1127 avail = sgl_flits;
1128 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1129 sgl_flits -= avail;
1130 ndesc--;
1131 if (!sgl_flits)
1132 break;
1133
1134 fp += avail;
1135 d++;
1136 sd->eop = 0;
1137 sd++;
1138 if (++pidx == q->size) {
1139 pidx = 0;
1140 gen ^= 1;
1141 d = q->desc;
1142 sd = q->sdesc;
1143 }
1144
1145 sd->skb = skb;
1146 wrp = (struct work_request_hdr *)d;
1147 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1148 V_WR_SGLSFLT(1)) | wr_hi;
1149 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1150 sgl_flits + 1)) |
1151 V_WR_GEN(gen)) | wr_lo;
1152 wr_gen2(d, gen);
1153 flits = 1;
1154 }
1155 sd->eop = 1;
1156 wrp->wr_hi |= htonl(F_WR_EOP);
1157 dma_wmb();
1158 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1159 wr_gen2((struct tx_desc *)wp, ogen);
1160 WARN_ON(ndesc != 0);
1161 }
1162}
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1179 const struct port_info *pi,
1180 unsigned int pidx, unsigned int gen,
1181 struct sge_txq *q, unsigned int ndesc,
1182 unsigned int compl, const dma_addr_t *addr)
1183{
1184 unsigned int flits, sgl_flits, cntrl, tso_info;
1185 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1186 struct tx_desc *d = &q->desc[pidx];
1187 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1188
1189 cpl->len = htonl(skb->len);
1190 cntrl = V_TXPKT_INTF(pi->port_id);
1191
1192 if (skb_vlan_tag_present(skb))
1193 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(skb_vlan_tag_get(skb));
1194
1195 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1196 if (tso_info) {
1197 int eth_type;
1198 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1199
1200 d->flit[2] = 0;
1201 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1202 hdr->cntrl = htonl(cntrl);
1203 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1204 CPL_ETH_II : CPL_ETH_II_VLAN;
1205 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1206 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1207 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1208 hdr->lso_info = htonl(tso_info);
1209 flits = 3;
1210 } else {
1211 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1212 cntrl |= F_TXPKT_IPCSUM_DIS;
1213 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1214 cpl->cntrl = htonl(cntrl);
1215
1216 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1217 q->sdesc[pidx].skb = NULL;
1218 if (!skb->data_len)
1219 skb_copy_from_linear_data(skb, &d->flit[2],
1220 skb->len);
1221 else
1222 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1223
1224 flits = (skb->len + 7) / 8 + 2;
1225 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1226 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1227 | F_WR_SOP | F_WR_EOP | compl);
1228 dma_wmb();
1229 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1230 V_WR_TID(q->token));
1231 wr_gen2(d, gen);
1232 dev_consume_skb_any(skb);
1233 return;
1234 }
1235
1236 flits = 2;
1237 }
1238
1239 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1240 sgl_flits = write_sgl(skb, sgp, skb->data, skb_headlen(skb), addr);
1241
1242 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1243 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1244 htonl(V_WR_TID(q->token)));
1245}
1246
1247static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1248 struct sge_qset *qs, struct sge_txq *q)
1249{
1250 netif_tx_stop_queue(txq);
1251 set_bit(TXQ_ETH, &qs->txq_stopped);
1252 q->stops++;
1253}
1254
1255
1256
1257
1258
1259
1260
1261
1262netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1263{
1264 int qidx;
1265 unsigned int ndesc, pidx, credits, gen, compl;
1266 const struct port_info *pi = netdev_priv(dev);
1267 struct adapter *adap = pi->adapter;
1268 struct netdev_queue *txq;
1269 struct sge_qset *qs;
1270 struct sge_txq *q;
1271 dma_addr_t addr[MAX_SKB_FRAGS + 1];
1272
1273
1274
1275
1276
1277 if (unlikely(skb->len < ETH_HLEN)) {
1278 dev_kfree_skb_any(skb);
1279 return NETDEV_TX_OK;
1280 }
1281
1282 qidx = skb_get_queue_mapping(skb);
1283 qs = &pi->qs[qidx];
1284 q = &qs->txq[TXQ_ETH];
1285 txq = netdev_get_tx_queue(dev, qidx);
1286
1287 reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1288
1289 credits = q->size - q->in_use;
1290 ndesc = calc_tx_descs(skb);
1291
1292 if (unlikely(credits < ndesc)) {
1293 t3_stop_tx_queue(txq, qs, q);
1294 dev_err(&adap->pdev->dev,
1295 "%s: Tx ring %u full while queue awake!\n",
1296 dev->name, q->cntxt_id & 7);
1297 return NETDEV_TX_BUSY;
1298 }
1299
1300
1301 if (skb->len > (WR_LEN - sizeof(struct cpl_tx_pkt))) {
1302 if (unlikely(map_skb(adap->pdev, skb, addr) < 0)) {
1303 dev_kfree_skb(skb);
1304 return NETDEV_TX_OK;
1305 }
1306 }
1307
1308 q->in_use += ndesc;
1309 if (unlikely(credits - ndesc < q->stop_thres)) {
1310 t3_stop_tx_queue(txq, qs, q);
1311
1312 if (should_restart_tx(q) &&
1313 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1314 q->restarts++;
1315 netif_tx_start_queue(txq);
1316 }
1317 }
1318
1319 gen = q->gen;
1320 q->unacked += ndesc;
1321 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1322 q->unacked &= 7;
1323 pidx = q->pidx;
1324 q->pidx += ndesc;
1325 if (q->pidx >= q->size) {
1326 q->pidx -= q->size;
1327 q->gen ^= 1;
1328 }
1329
1330
1331 if (skb->ip_summed == CHECKSUM_PARTIAL)
1332 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1333 if (skb_shinfo(skb)->gso_size)
1334 qs->port_stats[SGE_PSTAT_TSO]++;
1335 if (skb_vlan_tag_present(skb))
1336 qs->port_stats[SGE_PSTAT_VLANINS]++;
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362 if (likely(!skb_shared(skb)))
1363 skb_orphan(skb);
1364
1365 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl, addr);
1366 check_ring_tx_db(adap, q);
1367 return NETDEV_TX_OK;
1368}
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1383 unsigned int len, unsigned int gen)
1384{
1385 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1386 struct work_request_hdr *to = (struct work_request_hdr *)d;
1387
1388 if (likely(!skb->data_len))
1389 memcpy(&to[1], &from[1], len - sizeof(*from));
1390 else
1391 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1392
1393 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1394 V_WR_BCNTLFLT(len & 7));
1395 dma_wmb();
1396 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1397 V_WR_LEN((len + 7) / 8));
1398 wr_gen2(d, gen);
1399 kfree_skb(skb);
1400}
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1421 struct sk_buff *skb, unsigned int ndesc,
1422 unsigned int qid)
1423{
1424 if (unlikely(!skb_queue_empty(&q->sendq))) {
1425 addq_exit:__skb_queue_tail(&q->sendq, skb);
1426 return 1;
1427 }
1428 if (unlikely(q->size - q->in_use < ndesc)) {
1429 struct sge_qset *qs = txq_to_qset(q, qid);
1430
1431 set_bit(qid, &qs->txq_stopped);
1432 smp_mb__after_atomic();
1433
1434 if (should_restart_tx(q) &&
1435 test_and_clear_bit(qid, &qs->txq_stopped))
1436 return 2;
1437
1438 q->stops++;
1439 goto addq_exit;
1440 }
1441 return 0;
1442}
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1453{
1454 unsigned int reclaim = q->processed - q->cleaned;
1455
1456 q->in_use -= reclaim;
1457 q->cleaned += reclaim;
1458}
1459
1460static inline int immediate(const struct sk_buff *skb)
1461{
1462 return skb->len <= WR_LEN;
1463}
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1476 struct sk_buff *skb)
1477{
1478 int ret;
1479 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1480
1481 if (unlikely(!immediate(skb))) {
1482 WARN_ON(1);
1483 dev_kfree_skb(skb);
1484 return NET_XMIT_SUCCESS;
1485 }
1486
1487 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1488 wrp->wr_lo = htonl(V_WR_TID(q->token));
1489
1490 spin_lock(&q->lock);
1491 again:reclaim_completed_tx_imm(q);
1492
1493 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1494 if (unlikely(ret)) {
1495 if (ret == 1) {
1496 spin_unlock(&q->lock);
1497 return NET_XMIT_CN;
1498 }
1499 goto again;
1500 }
1501
1502 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1503
1504 q->in_use++;
1505 if (++q->pidx >= q->size) {
1506 q->pidx = 0;
1507 q->gen ^= 1;
1508 }
1509 spin_unlock(&q->lock);
1510 wmb();
1511 t3_write_reg(adap, A_SG_KDOORBELL,
1512 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1513 return NET_XMIT_SUCCESS;
1514}
1515
1516
1517
1518
1519
1520
1521
1522static void restart_ctrlq(struct work_struct *w)
1523{
1524 struct sk_buff *skb;
1525 struct sge_qset *qs = container_of(w, struct sge_qset,
1526 txq[TXQ_CTRL].qresume_task);
1527 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1528
1529 spin_lock(&q->lock);
1530 again:reclaim_completed_tx_imm(q);
1531
1532 while (q->in_use < q->size &&
1533 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1534
1535 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1536
1537 if (++q->pidx >= q->size) {
1538 q->pidx = 0;
1539 q->gen ^= 1;
1540 }
1541 q->in_use++;
1542 }
1543
1544 if (!skb_queue_empty(&q->sendq)) {
1545 set_bit(TXQ_CTRL, &qs->txq_stopped);
1546 smp_mb__after_atomic();
1547
1548 if (should_restart_tx(q) &&
1549 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1550 goto again;
1551 q->stops++;
1552 }
1553
1554 spin_unlock(&q->lock);
1555 wmb();
1556 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1557 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1558}
1559
1560
1561
1562
1563int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1564{
1565 int ret;
1566 local_bh_disable();
1567 ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1568 local_bh_enable();
1569
1570 return ret;
1571}
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581static void deferred_unmap_destructor(struct sk_buff *skb)
1582{
1583 int i;
1584 const dma_addr_t *p;
1585 const struct skb_shared_info *si;
1586 const struct deferred_unmap_info *dui;
1587
1588 dui = (struct deferred_unmap_info *)skb->head;
1589 p = dui->addr;
1590
1591 if (skb_tail_pointer(skb) - skb_transport_header(skb))
1592 dma_unmap_single(&dui->pdev->dev, *p++,
1593 skb_tail_pointer(skb) - skb_transport_header(skb),
1594 DMA_TO_DEVICE);
1595
1596 si = skb_shinfo(skb);
1597 for (i = 0; i < si->nr_frags; i++)
1598 dma_unmap_page(&dui->pdev->dev, *p++,
1599 skb_frag_size(&si->frags[i]), DMA_TO_DEVICE);
1600}
1601
1602static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1603 const struct sg_ent *sgl, int sgl_flits)
1604{
1605 dma_addr_t *p;
1606 struct deferred_unmap_info *dui;
1607
1608 dui = (struct deferred_unmap_info *)skb->head;
1609 dui->pdev = pdev;
1610 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1611 *p++ = be64_to_cpu(sgl->addr[0]);
1612 *p++ = be64_to_cpu(sgl->addr[1]);
1613 }
1614 if (sgl_flits)
1615 *p = be64_to_cpu(sgl->addr[0]);
1616}
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1632 struct sge_txq *q, unsigned int pidx,
1633 unsigned int gen, unsigned int ndesc,
1634 const dma_addr_t *addr)
1635{
1636 unsigned int sgl_flits, flits;
1637 struct work_request_hdr *from;
1638 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1639 struct tx_desc *d = &q->desc[pidx];
1640
1641 if (immediate(skb)) {
1642 q->sdesc[pidx].skb = NULL;
1643 write_imm(d, skb, skb->len, gen);
1644 return;
1645 }
1646
1647
1648
1649 from = (struct work_request_hdr *)skb->data;
1650 memcpy(&d->flit[1], &from[1],
1651 skb_transport_offset(skb) - sizeof(*from));
1652
1653 flits = skb_transport_offset(skb) / 8;
1654 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1655 sgl_flits = write_sgl(skb, sgp, skb_transport_header(skb),
1656 skb_tail_pointer(skb) - skb_transport_header(skb),
1657 addr);
1658 if (need_skb_unmap()) {
1659 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1660 skb->destructor = deferred_unmap_destructor;
1661 }
1662
1663 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1664 gen, from->wr_hi, from->wr_lo);
1665}
1666
1667
1668
1669
1670
1671
1672
1673
1674static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1675{
1676 unsigned int flits, cnt;
1677
1678 if (skb->len <= WR_LEN)
1679 return 1;
1680
1681 flits = skb_transport_offset(skb) / 8;
1682 cnt = skb_shinfo(skb)->nr_frags;
1683 if (skb_tail_pointer(skb) != skb_transport_header(skb))
1684 cnt++;
1685 return flits_to_desc(flits + sgl_len(cnt));
1686}
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1697 struct sk_buff *skb)
1698{
1699 int ret;
1700 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1701
1702 spin_lock(&q->lock);
1703again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1704
1705 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1706 if (unlikely(ret)) {
1707 if (ret == 1) {
1708 skb->priority = ndesc;
1709 spin_unlock(&q->lock);
1710 return NET_XMIT_CN;
1711 }
1712 goto again;
1713 }
1714
1715 if (!immediate(skb) &&
1716 map_skb(adap->pdev, skb, (dma_addr_t *)skb->head)) {
1717 spin_unlock(&q->lock);
1718 return NET_XMIT_SUCCESS;
1719 }
1720
1721 gen = q->gen;
1722 q->in_use += ndesc;
1723 pidx = q->pidx;
1724 q->pidx += ndesc;
1725 if (q->pidx >= q->size) {
1726 q->pidx -= q->size;
1727 q->gen ^= 1;
1728 }
1729 spin_unlock(&q->lock);
1730
1731 write_ofld_wr(adap, skb, q, pidx, gen, ndesc, (dma_addr_t *)skb->head);
1732 check_ring_tx_db(adap, q);
1733 return NET_XMIT_SUCCESS;
1734}
1735
1736
1737
1738
1739
1740
1741
1742static void restart_offloadq(struct work_struct *w)
1743{
1744 struct sk_buff *skb;
1745 struct sge_qset *qs = container_of(w, struct sge_qset,
1746 txq[TXQ_OFLD].qresume_task);
1747 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1748 const struct port_info *pi = netdev_priv(qs->netdev);
1749 struct adapter *adap = pi->adapter;
1750 unsigned int written = 0;
1751
1752 spin_lock(&q->lock);
1753again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1754
1755 while ((skb = skb_peek(&q->sendq)) != NULL) {
1756 unsigned int gen, pidx;
1757 unsigned int ndesc = skb->priority;
1758
1759 if (unlikely(q->size - q->in_use < ndesc)) {
1760 set_bit(TXQ_OFLD, &qs->txq_stopped);
1761 smp_mb__after_atomic();
1762
1763 if (should_restart_tx(q) &&
1764 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1765 goto again;
1766 q->stops++;
1767 break;
1768 }
1769
1770 if (!immediate(skb) &&
1771 map_skb(adap->pdev, skb, (dma_addr_t *)skb->head))
1772 break;
1773
1774 gen = q->gen;
1775 q->in_use += ndesc;
1776 pidx = q->pidx;
1777 q->pidx += ndesc;
1778 written += ndesc;
1779 if (q->pidx >= q->size) {
1780 q->pidx -= q->size;
1781 q->gen ^= 1;
1782 }
1783 __skb_unlink(skb, &q->sendq);
1784 spin_unlock(&q->lock);
1785
1786 write_ofld_wr(adap, skb, q, pidx, gen, ndesc,
1787 (dma_addr_t *)skb->head);
1788 spin_lock(&q->lock);
1789 }
1790 spin_unlock(&q->lock);
1791
1792#if USE_GTS
1793 set_bit(TXQ_RUNNING, &q->flags);
1794 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1795#endif
1796 wmb();
1797 if (likely(written))
1798 t3_write_reg(adap, A_SG_KDOORBELL,
1799 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1800}
1801
1802
1803
1804
1805
1806
1807
1808
1809static inline int queue_set(const struct sk_buff *skb)
1810{
1811 return skb->priority >> 1;
1812}
1813
1814
1815
1816
1817
1818
1819
1820
1821static inline int is_ctrl_pkt(const struct sk_buff *skb)
1822{
1823 return skb->priority & 1;
1824}
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1836{
1837 struct adapter *adap = tdev2adap(tdev);
1838 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1839
1840 if (unlikely(is_ctrl_pkt(skb)))
1841 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1842
1843 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1844}
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1856{
1857 int was_empty = skb_queue_empty(&q->rx_queue);
1858
1859 __skb_queue_tail(&q->rx_queue, skb);
1860
1861 if (was_empty) {
1862 struct sge_qset *qs = rspq_to_qset(q);
1863
1864 napi_schedule(&qs->napi);
1865 }
1866}
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877static inline void deliver_partial_bundle(struct t3cdev *tdev,
1878 struct sge_rspq *q,
1879 struct sk_buff *skbs[], int n)
1880{
1881 if (n) {
1882 q->offload_bundles++;
1883 tdev->recv(tdev, skbs, n);
1884 }
1885}
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898static int ofld_poll(struct napi_struct *napi, int budget)
1899{
1900 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1901 struct sge_rspq *q = &qs->rspq;
1902 struct adapter *adapter = qs->adap;
1903 int work_done = 0;
1904
1905 while (work_done < budget) {
1906 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1907 struct sk_buff_head queue;
1908 int ngathered;
1909
1910 spin_lock_irq(&q->lock);
1911 __skb_queue_head_init(&queue);
1912 skb_queue_splice_init(&q->rx_queue, &queue);
1913 if (skb_queue_empty(&queue)) {
1914 napi_complete_done(napi, work_done);
1915 spin_unlock_irq(&q->lock);
1916 return work_done;
1917 }
1918 spin_unlock_irq(&q->lock);
1919
1920 ngathered = 0;
1921 skb_queue_walk_safe(&queue, skb, tmp) {
1922 if (work_done >= budget)
1923 break;
1924 work_done++;
1925
1926 __skb_unlink(skb, &queue);
1927 prefetch(skb->data);
1928 skbs[ngathered] = skb;
1929 if (++ngathered == RX_BUNDLE_SIZE) {
1930 q->offload_bundles++;
1931 adapter->tdev.recv(&adapter->tdev, skbs,
1932 ngathered);
1933 ngathered = 0;
1934 }
1935 }
1936 if (!skb_queue_empty(&queue)) {
1937
1938 spin_lock_irq(&q->lock);
1939 skb_queue_splice(&queue, &q->rx_queue);
1940 spin_unlock_irq(&q->lock);
1941 }
1942 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1943 }
1944
1945 return work_done;
1946}
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1960 struct sk_buff *skb, struct sk_buff *rx_gather[],
1961 unsigned int gather_idx)
1962{
1963 skb_reset_mac_header(skb);
1964 skb_reset_network_header(skb);
1965 skb_reset_transport_header(skb);
1966
1967 if (rq->polling) {
1968 rx_gather[gather_idx++] = skb;
1969 if (gather_idx == RX_BUNDLE_SIZE) {
1970 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1971 gather_idx = 0;
1972 rq->offload_bundles++;
1973 }
1974 } else
1975 offload_enqueue(rq, skb);
1976
1977 return gather_idx;
1978}
1979
1980
1981
1982
1983
1984
1985
1986
1987static void restart_tx(struct sge_qset *qs)
1988{
1989 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1990 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1991 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1992 qs->txq[TXQ_ETH].restarts++;
1993 if (netif_running(qs->netdev))
1994 netif_tx_wake_queue(qs->tx_q);
1995 }
1996
1997 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1998 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1999 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
2000 qs->txq[TXQ_OFLD].restarts++;
2001
2002
2003 queue_work(cxgb3_wq, &qs->txq[TXQ_OFLD].qresume_task);
2004 }
2005 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
2006 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
2007 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
2008 qs->txq[TXQ_CTRL].restarts++;
2009
2010
2011 queue_work(cxgb3_wq, &qs->txq[TXQ_CTRL].qresume_task);
2012 }
2013}
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb)
2024{
2025 struct net_device *dev = skb->dev;
2026 struct arphdr *arp;
2027 unsigned char *arp_ptr;
2028 unsigned char *sha;
2029 __be32 sip, tip;
2030
2031 if (!dev)
2032 return;
2033
2034 skb_reset_network_header(skb);
2035 arp = arp_hdr(skb);
2036
2037 if (arp->ar_op != htons(ARPOP_REQUEST))
2038 return;
2039
2040 arp_ptr = (unsigned char *)(arp + 1);
2041 sha = arp_ptr;
2042 arp_ptr += dev->addr_len;
2043 memcpy(&sip, arp_ptr, sizeof(sip));
2044 arp_ptr += sizeof(sip);
2045 arp_ptr += dev->addr_len;
2046 memcpy(&tip, arp_ptr, sizeof(tip));
2047
2048 if (tip != pi->iscsi_ipv4addr)
2049 return;
2050
2051 arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
2052 pi->iscsic.mac_addr, sha);
2053
2054}
2055
2056static inline int is_arp(struct sk_buff *skb)
2057{
2058 return skb->protocol == htons(ETH_P_ARP);
2059}
2060
2061static void cxgb3_process_iscsi_prov_pack(struct port_info *pi,
2062 struct sk_buff *skb)
2063{
2064 if (is_arp(skb)) {
2065 cxgb3_arp_process(pi, skb);
2066 return;
2067 }
2068
2069 if (pi->iscsic.recv)
2070 pi->iscsic.recv(pi, skb);
2071
2072}
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2087 struct sk_buff *skb, int pad, int lro)
2088{
2089 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2090 struct sge_qset *qs = rspq_to_qset(rq);
2091 struct port_info *pi;
2092
2093 skb_pull(skb, sizeof(*p) + pad);
2094 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2095 pi = netdev_priv(skb->dev);
2096 if ((skb->dev->features & NETIF_F_RXCSUM) && p->csum_valid &&
2097 p->csum == htons(0xffff) && !p->fragment) {
2098 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2099 skb->ip_summed = CHECKSUM_UNNECESSARY;
2100 } else
2101 skb_checksum_none_assert(skb);
2102 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2103
2104 if (p->vlan_valid) {
2105 qs->port_stats[SGE_PSTAT_VLANEX]++;
2106 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(p->vlan));
2107 }
2108 if (rq->polling) {
2109 if (lro)
2110 napi_gro_receive(&qs->napi, skb);
2111 else {
2112 if (unlikely(pi->iscsic.flags))
2113 cxgb3_process_iscsi_prov_pack(pi, skb);
2114 netif_receive_skb(skb);
2115 }
2116 } else
2117 netif_rx(skb);
2118}
2119
2120static inline int is_eth_tcp(u32 rss)
2121{
2122 return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2123}
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2137 struct sge_fl *fl, int len, int complete)
2138{
2139 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2140 struct port_info *pi = netdev_priv(qs->netdev);
2141 struct sk_buff *skb = NULL;
2142 struct cpl_rx_pkt *cpl;
2143 skb_frag_t *rx_frag;
2144 int nr_frags;
2145 int offset = 0;
2146
2147 if (!qs->nomem) {
2148 skb = napi_get_frags(&qs->napi);
2149 qs->nomem = !skb;
2150 }
2151
2152 fl->credits--;
2153
2154 dma_sync_single_for_cpu(&adap->pdev->dev,
2155 dma_unmap_addr(sd, dma_addr),
2156 fl->buf_size - SGE_PG_RSVD, DMA_FROM_DEVICE);
2157
2158 (*sd->pg_chunk.p_cnt)--;
2159 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
2160 dma_unmap_page(&adap->pdev->dev, sd->pg_chunk.mapping,
2161 fl->alloc_size, DMA_FROM_DEVICE);
2162
2163 if (!skb) {
2164 put_page(sd->pg_chunk.page);
2165 if (complete)
2166 qs->nomem = 0;
2167 return;
2168 }
2169
2170 rx_frag = skb_shinfo(skb)->frags;
2171 nr_frags = skb_shinfo(skb)->nr_frags;
2172
2173 if (!nr_frags) {
2174 offset = 2 + sizeof(struct cpl_rx_pkt);
2175 cpl = qs->lro_va = sd->pg_chunk.va + 2;
2176
2177 if ((qs->netdev->features & NETIF_F_RXCSUM) &&
2178 cpl->csum_valid && cpl->csum == htons(0xffff)) {
2179 skb->ip_summed = CHECKSUM_UNNECESSARY;
2180 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2181 } else
2182 skb->ip_summed = CHECKSUM_NONE;
2183 } else
2184 cpl = qs->lro_va;
2185
2186 len -= offset;
2187
2188 rx_frag += nr_frags;
2189 __skb_frag_set_page(rx_frag, sd->pg_chunk.page);
2190 skb_frag_off_set(rx_frag, sd->pg_chunk.offset + offset);
2191 skb_frag_size_set(rx_frag, len);
2192
2193 skb->len += len;
2194 skb->data_len += len;
2195 skb->truesize += len;
2196 skb_shinfo(skb)->nr_frags++;
2197
2198 if (!complete)
2199 return;
2200
2201 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2202
2203 if (cpl->vlan_valid) {
2204 qs->port_stats[SGE_PSTAT_VLANEX]++;
2205 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cpl->vlan));
2206 }
2207 napi_gro_frags(&qs->napi);
2208}
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2220{
2221 unsigned int credits;
2222
2223#if USE_GTS
2224 if (flags & F_RSPD_TXQ0_GTS)
2225 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2226#endif
2227
2228 credits = G_RSPD_TXQ0_CR(flags);
2229 if (credits)
2230 qs->txq[TXQ_ETH].processed += credits;
2231
2232 credits = G_RSPD_TXQ2_CR(flags);
2233 if (credits)
2234 qs->txq[TXQ_CTRL].processed += credits;
2235
2236# if USE_GTS
2237 if (flags & F_RSPD_TXQ1_GTS)
2238 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2239# endif
2240 credits = G_RSPD_TXQ1_CR(flags);
2241 if (credits)
2242 qs->txq[TXQ_OFLD].processed += credits;
2243}
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2256 unsigned int sleeping)
2257{
2258 if (sleeping & F_RSPD_TXQ0_GTS) {
2259 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2260
2261 if (txq->cleaned + txq->in_use != txq->processed &&
2262 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2263 set_bit(TXQ_RUNNING, &txq->flags);
2264 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2265 V_EGRCNTX(txq->cntxt_id));
2266 }
2267 }
2268
2269 if (sleeping & F_RSPD_TXQ1_GTS) {
2270 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2271
2272 if (txq->cleaned + txq->in_use != txq->processed &&
2273 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2274 set_bit(TXQ_RUNNING, &txq->flags);
2275 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2276 V_EGRCNTX(txq->cntxt_id));
2277 }
2278 }
2279}
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289static inline int is_new_response(const struct rsp_desc *r,
2290 const struct sge_rspq *q)
2291{
2292 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2293}
2294
2295static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2296{
2297 q->pg_skb = NULL;
2298 q->rx_recycle_buf = 0;
2299}
2300
2301#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2302#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2303 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2304 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2305 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2306
2307
2308#define NOMEM_INTR_DELAY 2500
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325static int process_responses(struct adapter *adap, struct sge_qset *qs,
2326 int budget)
2327{
2328 struct sge_rspq *q = &qs->rspq;
2329 struct rsp_desc *r = &q->desc[q->cidx];
2330 int budget_left = budget;
2331 unsigned int sleeping = 0;
2332 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2333 int ngathered = 0;
2334
2335 q->next_holdoff = q->holdoff_tmr;
2336
2337 while (likely(budget_left && is_new_response(r, q))) {
2338 int packet_complete, eth, ethpad = 2;
2339 int lro = !!(qs->netdev->features & NETIF_F_GRO);
2340 struct sk_buff *skb = NULL;
2341 u32 len, flags;
2342 __be32 rss_hi, rss_lo;
2343
2344 dma_rmb();
2345 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2346 rss_hi = *(const __be32 *)r;
2347 rss_lo = r->rss_hdr.rss_hash_val;
2348 flags = ntohl(r->flags);
2349
2350 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2351 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2352 if (!skb)
2353 goto no_mem;
2354
2355 __skb_put_data(skb, r, AN_PKT_SIZE);
2356 skb->data[0] = CPL_ASYNC_NOTIF;
2357 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2358 q->async_notif++;
2359 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2360 skb = get_imm_packet(r);
2361 if (unlikely(!skb)) {
2362no_mem:
2363 q->next_holdoff = NOMEM_INTR_DELAY;
2364 q->nomem++;
2365
2366 budget_left--;
2367 break;
2368 }
2369 q->imm_data++;
2370 ethpad = 0;
2371 } else if ((len = ntohl(r->len_cq)) != 0) {
2372 struct sge_fl *fl;
2373
2374 lro &= eth && is_eth_tcp(rss_hi);
2375
2376 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2377 if (fl->use_pages) {
2378 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2379
2380 net_prefetch(addr);
2381 __refill_fl(adap, fl);
2382 if (lro > 0) {
2383 lro_add_page(adap, qs, fl,
2384 G_RSPD_LEN(len),
2385 flags & F_RSPD_EOP);
2386 goto next_fl;
2387 }
2388
2389 skb = get_packet_pg(adap, fl, q,
2390 G_RSPD_LEN(len),
2391 eth ?
2392 SGE_RX_DROP_THRES : 0);
2393 q->pg_skb = skb;
2394 } else
2395 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2396 eth ? SGE_RX_DROP_THRES : 0);
2397 if (unlikely(!skb)) {
2398 if (!eth)
2399 goto no_mem;
2400 q->rx_drops++;
2401 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2402 __skb_pull(skb, 2);
2403next_fl:
2404 if (++fl->cidx == fl->size)
2405 fl->cidx = 0;
2406 } else
2407 q->pure_rsps++;
2408
2409 if (flags & RSPD_CTRL_MASK) {
2410 sleeping |= flags & RSPD_GTS_MASK;
2411 handle_rsp_cntrl_info(qs, flags);
2412 }
2413
2414 r++;
2415 if (unlikely(++q->cidx == q->size)) {
2416 q->cidx = 0;
2417 q->gen ^= 1;
2418 r = q->desc;
2419 }
2420 prefetch(r);
2421
2422 if (++q->credits >= (q->size / 4)) {
2423 refill_rspq(adap, q, q->credits);
2424 q->credits = 0;
2425 }
2426
2427 packet_complete = flags &
2428 (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2429 F_RSPD_ASYNC_NOTIF);
2430
2431 if (skb != NULL && packet_complete) {
2432 if (eth)
2433 rx_eth(adap, q, skb, ethpad, lro);
2434 else {
2435 q->offload_pkts++;
2436
2437 skb->csum = rss_hi;
2438 skb->priority = rss_lo;
2439 ngathered = rx_offload(&adap->tdev, q, skb,
2440 offload_skbs,
2441 ngathered);
2442 }
2443
2444 if (flags & F_RSPD_EOP)
2445 clear_rspq_bufstate(q);
2446 }
2447 --budget_left;
2448 }
2449
2450 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2451
2452 if (sleeping)
2453 check_ring_db(adap, qs, sleeping);
2454
2455 smp_mb();
2456 if (unlikely(qs->txq_stopped != 0))
2457 restart_tx(qs);
2458
2459 budget -= budget_left;
2460 return budget;
2461}
2462
2463static inline int is_pure_response(const struct rsp_desc *r)
2464{
2465 __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2466
2467 return (n | r->len_cq) == 0;
2468}
2469
2470
2471
2472
2473
2474
2475
2476
2477static int napi_rx_handler(struct napi_struct *napi, int budget)
2478{
2479 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2480 struct adapter *adap = qs->adap;
2481 int work_done = process_responses(adap, qs, budget);
2482
2483 if (likely(work_done < budget)) {
2484 napi_complete_done(napi, work_done);
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2501 V_NEWTIMER(qs->rspq.next_holdoff) |
2502 V_NEWINDEX(qs->rspq.cidx));
2503 }
2504 return work_done;
2505}
2506
2507
2508
2509
2510static inline int napi_is_scheduled(struct napi_struct *napi)
2511{
2512 return test_bit(NAPI_STATE_SCHED, &napi->state);
2513}
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2530 struct rsp_desc *r)
2531{
2532 struct sge_rspq *q = &qs->rspq;
2533 unsigned int sleeping = 0;
2534
2535 do {
2536 u32 flags = ntohl(r->flags);
2537
2538 r++;
2539 if (unlikely(++q->cidx == q->size)) {
2540 q->cidx = 0;
2541 q->gen ^= 1;
2542 r = q->desc;
2543 }
2544 prefetch(r);
2545
2546 if (flags & RSPD_CTRL_MASK) {
2547 sleeping |= flags & RSPD_GTS_MASK;
2548 handle_rsp_cntrl_info(qs, flags);
2549 }
2550
2551 q->pure_rsps++;
2552 if (++q->credits >= (q->size / 4)) {
2553 refill_rspq(adap, q, q->credits);
2554 q->credits = 0;
2555 }
2556 if (!is_new_response(r, q))
2557 break;
2558 dma_rmb();
2559 } while (is_pure_response(r));
2560
2561 if (sleeping)
2562 check_ring_db(adap, qs, sleeping);
2563
2564 smp_mb();
2565 if (unlikely(qs->txq_stopped != 0))
2566 restart_tx(qs);
2567
2568 return is_new_response(r, q);
2569}
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2587{
2588 struct sge_qset *qs = rspq_to_qset(q);
2589 struct rsp_desc *r = &q->desc[q->cidx];
2590
2591 if (!is_new_response(r, q))
2592 return -1;
2593 dma_rmb();
2594 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2595 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2596 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2597 return 0;
2598 }
2599 napi_schedule(&qs->napi);
2600 return 1;
2601}
2602
2603
2604
2605
2606
2607static irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2608{
2609 struct sge_qset *qs = cookie;
2610 struct adapter *adap = qs->adap;
2611 struct sge_rspq *q = &qs->rspq;
2612
2613 spin_lock(&q->lock);
2614 if (process_responses(adap, qs, -1) == 0)
2615 q->unhandled_irqs++;
2616 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2617 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2618 spin_unlock(&q->lock);
2619 return IRQ_HANDLED;
2620}
2621
2622
2623
2624
2625
2626static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2627{
2628 struct sge_qset *qs = cookie;
2629 struct sge_rspq *q = &qs->rspq;
2630
2631 spin_lock(&q->lock);
2632
2633 if (handle_responses(qs->adap, q) < 0)
2634 q->unhandled_irqs++;
2635 spin_unlock(&q->lock);
2636 return IRQ_HANDLED;
2637}
2638
2639
2640
2641
2642
2643
2644
2645static irqreturn_t t3_intr_msi(int irq, void *cookie)
2646{
2647 int new_packets = 0;
2648 struct adapter *adap = cookie;
2649 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2650
2651 spin_lock(&q->lock);
2652
2653 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2654 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2655 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2656 new_packets = 1;
2657 }
2658
2659 if (adap->params.nports == 2 &&
2660 process_responses(adap, &adap->sge.qs[1], -1)) {
2661 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2662
2663 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2664 V_NEWTIMER(q1->next_holdoff) |
2665 V_NEWINDEX(q1->cidx));
2666 new_packets = 1;
2667 }
2668
2669 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2670 q->unhandled_irqs++;
2671
2672 spin_unlock(&q->lock);
2673 return IRQ_HANDLED;
2674}
2675
2676static int rspq_check_napi(struct sge_qset *qs)
2677{
2678 struct sge_rspq *q = &qs->rspq;
2679
2680 if (!napi_is_scheduled(&qs->napi) &&
2681 is_new_response(&q->desc[q->cidx], q)) {
2682 napi_schedule(&qs->napi);
2683 return 1;
2684 }
2685 return 0;
2686}
2687
2688
2689
2690
2691
2692
2693
2694
2695static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2696{
2697 int new_packets;
2698 struct adapter *adap = cookie;
2699 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2700
2701 spin_lock(&q->lock);
2702
2703 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2704 if (adap->params.nports == 2)
2705 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2706 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2707 q->unhandled_irqs++;
2708
2709 spin_unlock(&q->lock);
2710 return IRQ_HANDLED;
2711}
2712
2713
2714
2715
2716static inline int process_responses_gts(struct adapter *adap,
2717 struct sge_rspq *rq)
2718{
2719 int work;
2720
2721 work = process_responses(adap, rspq_to_qset(rq), -1);
2722 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2723 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2724 return work;
2725}
2726
2727
2728
2729
2730
2731
2732
2733static irqreturn_t t3_intr(int irq, void *cookie)
2734{
2735 int work_done, w0, w1;
2736 struct adapter *adap = cookie;
2737 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2738 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2739
2740 spin_lock(&q0->lock);
2741
2742 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2743 w1 = adap->params.nports == 2 &&
2744 is_new_response(&q1->desc[q1->cidx], q1);
2745
2746 if (likely(w0 | w1)) {
2747 t3_write_reg(adap, A_PL_CLI, 0);
2748 t3_read_reg(adap, A_PL_CLI);
2749
2750 if (likely(w0))
2751 process_responses_gts(adap, q0);
2752
2753 if (w1)
2754 process_responses_gts(adap, q1);
2755
2756 work_done = w0 | w1;
2757 } else
2758 work_done = t3_slow_intr_handler(adap);
2759
2760 spin_unlock(&q0->lock);
2761 return IRQ_RETVAL(work_done != 0);
2762}
2763
2764
2765
2766
2767
2768
2769
2770
2771static irqreturn_t t3b_intr(int irq, void *cookie)
2772{
2773 u32 map;
2774 struct adapter *adap = cookie;
2775 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2776
2777 t3_write_reg(adap, A_PL_CLI, 0);
2778 map = t3_read_reg(adap, A_SG_DATA_INTR);
2779
2780 if (unlikely(!map))
2781 return IRQ_NONE;
2782
2783 spin_lock(&q0->lock);
2784
2785 if (unlikely(map & F_ERRINTR))
2786 t3_slow_intr_handler(adap);
2787
2788 if (likely(map & 1))
2789 process_responses_gts(adap, q0);
2790
2791 if (map & 2)
2792 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2793
2794 spin_unlock(&q0->lock);
2795 return IRQ_HANDLED;
2796}
2797
2798
2799
2800
2801
2802
2803
2804
2805static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2806{
2807 u32 map;
2808 struct adapter *adap = cookie;
2809 struct sge_qset *qs0 = &adap->sge.qs[0];
2810 struct sge_rspq *q0 = &qs0->rspq;
2811
2812 t3_write_reg(adap, A_PL_CLI, 0);
2813 map = t3_read_reg(adap, A_SG_DATA_INTR);
2814
2815 if (unlikely(!map))
2816 return IRQ_NONE;
2817
2818 spin_lock(&q0->lock);
2819
2820 if (unlikely(map & F_ERRINTR))
2821 t3_slow_intr_handler(adap);
2822
2823 if (likely(map & 1))
2824 napi_schedule(&qs0->napi);
2825
2826 if (map & 2)
2827 napi_schedule(&adap->sge.qs[1].napi);
2828
2829 spin_unlock(&q0->lock);
2830 return IRQ_HANDLED;
2831}
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2843{
2844 if (adap->flags & USING_MSIX)
2845 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2846 if (adap->flags & USING_MSI)
2847 return polling ? t3_intr_msi_napi : t3_intr_msi;
2848 if (adap->params.rev > 0)
2849 return polling ? t3b_intr_napi : t3b_intr;
2850 return t3_intr;
2851}
2852
2853#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2854 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2855 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2856 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2857 F_HIRCQPARITYERROR)
2858#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2859#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2860 F_RSPQDISABLED)
2861
2862
2863
2864
2865
2866
2867
2868void t3_sge_err_intr_handler(struct adapter *adapter)
2869{
2870 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2871 ~F_FLEMPTY;
2872
2873 if (status & SGE_PARERR)
2874 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2875 status & SGE_PARERR);
2876 if (status & SGE_FRAMINGERR)
2877 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2878 status & SGE_FRAMINGERR);
2879
2880 if (status & F_RSPQCREDITOVERFOW)
2881 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2882
2883 if (status & F_RSPQDISABLED) {
2884 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2885
2886 CH_ALERT(adapter,
2887 "packet delivered to disabled response queue "
2888 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2889 }
2890
2891 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2892 queue_work(cxgb3_wq, &adapter->db_drop_task);
2893
2894 if (status & (F_HIPRIORITYDBFULL | F_LOPRIORITYDBFULL))
2895 queue_work(cxgb3_wq, &adapter->db_full_task);
2896
2897 if (status & (F_HIPRIORITYDBEMPTY | F_LOPRIORITYDBEMPTY))
2898 queue_work(cxgb3_wq, &adapter->db_empty_task);
2899
2900 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2901 if (status & SGE_FATALERR)
2902 t3_fatal_err(adapter);
2903}
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923static void sge_timer_tx(struct timer_list *t)
2924{
2925 struct sge_qset *qs = from_timer(qs, t, tx_reclaim_timer);
2926 struct port_info *pi = netdev_priv(qs->netdev);
2927 struct adapter *adap = pi->adapter;
2928 unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2929 unsigned long next_period;
2930
2931 if (__netif_tx_trylock(qs->tx_q)) {
2932 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2933 TX_RECLAIM_TIMER_CHUNK);
2934 __netif_tx_unlock(qs->tx_q);
2935 }
2936
2937 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2938 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2939 TX_RECLAIM_TIMER_CHUNK);
2940 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2941 }
2942
2943 next_period = TX_RECLAIM_PERIOD >>
2944 (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2945 TX_RECLAIM_TIMER_CHUNK);
2946 mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2947}
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963static void sge_timer_rx(struct timer_list *t)
2964{
2965 spinlock_t *lock;
2966 struct sge_qset *qs = from_timer(qs, t, rx_reclaim_timer);
2967 struct port_info *pi = netdev_priv(qs->netdev);
2968 struct adapter *adap = pi->adapter;
2969 u32 status;
2970
2971 lock = adap->params.rev > 0 ?
2972 &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2973
2974 if (!spin_trylock_irq(lock))
2975 goto out;
2976
2977 if (napi_is_scheduled(&qs->napi))
2978 goto unlock;
2979
2980 if (adap->params.rev < 4) {
2981 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2982
2983 if (status & (1 << qs->rspq.cntxt_id)) {
2984 qs->rspq.starved++;
2985 if (qs->rspq.credits) {
2986 qs->rspq.credits--;
2987 refill_rspq(adap, &qs->rspq, 1);
2988 qs->rspq.restarted++;
2989 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2990 1 << qs->rspq.cntxt_id);
2991 }
2992 }
2993 }
2994
2995 if (qs->fl[0].credits < qs->fl[0].size)
2996 __refill_fl(adap, &qs->fl[0]);
2997 if (qs->fl[1].credits < qs->fl[1].size)
2998 __refill_fl(adap, &qs->fl[1]);
2999
3000unlock:
3001 spin_unlock_irq(lock);
3002out:
3003 mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3004}
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
3015{
3016 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);
3017 qs->rspq.polling = p->polling;
3018 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
3019}
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
3038 int irq_vec_idx, const struct qset_params *p,
3039 int ntxq, struct net_device *dev,
3040 struct netdev_queue *netdevq)
3041{
3042 int i, avail, ret = -ENOMEM;
3043 struct sge_qset *q = &adapter->sge.qs[id];
3044
3045 init_qset_cntxt(q, id);
3046 timer_setup(&q->tx_reclaim_timer, sge_timer_tx, 0);
3047 timer_setup(&q->rx_reclaim_timer, sge_timer_rx, 0);
3048
3049 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
3050 sizeof(struct rx_desc),
3051 sizeof(struct rx_sw_desc),
3052 &q->fl[0].phys_addr, &q->fl[0].sdesc);
3053 if (!q->fl[0].desc)
3054 goto err;
3055
3056 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
3057 sizeof(struct rx_desc),
3058 sizeof(struct rx_sw_desc),
3059 &q->fl[1].phys_addr, &q->fl[1].sdesc);
3060 if (!q->fl[1].desc)
3061 goto err;
3062
3063 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
3064 sizeof(struct rsp_desc), 0,
3065 &q->rspq.phys_addr, NULL);
3066 if (!q->rspq.desc)
3067 goto err;
3068
3069 for (i = 0; i < ntxq; ++i) {
3070
3071
3072
3073
3074 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3075
3076 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3077 sizeof(struct tx_desc), sz,
3078 &q->txq[i].phys_addr,
3079 &q->txq[i].sdesc);
3080 if (!q->txq[i].desc)
3081 goto err;
3082
3083 q->txq[i].gen = 1;
3084 q->txq[i].size = p->txq_size[i];
3085 spin_lock_init(&q->txq[i].lock);
3086 skb_queue_head_init(&q->txq[i].sendq);
3087 }
3088
3089 INIT_WORK(&q->txq[TXQ_OFLD].qresume_task, restart_offloadq);
3090 INIT_WORK(&q->txq[TXQ_CTRL].qresume_task, restart_ctrlq);
3091
3092 q->fl[0].gen = q->fl[1].gen = 1;
3093 q->fl[0].size = p->fl_size;
3094 q->fl[1].size = p->jumbo_size;
3095
3096 q->rspq.gen = 1;
3097 q->rspq.size = p->rspq_size;
3098 spin_lock_init(&q->rspq.lock);
3099 skb_queue_head_init(&q->rspq.rx_queue);
3100
3101 q->txq[TXQ_ETH].stop_thres = nports *
3102 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3103
3104#if FL0_PG_CHUNK_SIZE > 0
3105 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3106#else
3107 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3108#endif
3109#if FL1_PG_CHUNK_SIZE > 0
3110 q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3111#else
3112 q->fl[1].buf_size = is_offload(adapter) ?
3113 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3114 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3115#endif
3116
3117 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3118 q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3119 q->fl[0].order = FL0_PG_ORDER;
3120 q->fl[1].order = FL1_PG_ORDER;
3121 q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3122 q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3123
3124 spin_lock_irq(&adapter->sge.reg_lock);
3125
3126
3127 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3128 q->rspq.phys_addr, q->rspq.size,
3129 q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3130 if (ret)
3131 goto err_unlock;
3132
3133 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3134 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3135 q->fl[i].phys_addr, q->fl[i].size,
3136 q->fl[i].buf_size - SGE_PG_RSVD,
3137 p->cong_thres, 1, 0);
3138 if (ret)
3139 goto err_unlock;
3140 }
3141
3142 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3143 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3144 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3145 1, 0);
3146 if (ret)
3147 goto err_unlock;
3148
3149 if (ntxq > 1) {
3150 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3151 USE_GTS, SGE_CNTXT_OFLD, id,
3152 q->txq[TXQ_OFLD].phys_addr,
3153 q->txq[TXQ_OFLD].size, 0, 1, 0);
3154 if (ret)
3155 goto err_unlock;
3156 }
3157
3158 if (ntxq > 2) {
3159 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3160 SGE_CNTXT_CTRL, id,
3161 q->txq[TXQ_CTRL].phys_addr,
3162 q->txq[TXQ_CTRL].size,
3163 q->txq[TXQ_CTRL].token, 1, 0);
3164 if (ret)
3165 goto err_unlock;
3166 }
3167
3168 spin_unlock_irq(&adapter->sge.reg_lock);
3169
3170 q->adap = adapter;
3171 q->netdev = dev;
3172 q->tx_q = netdevq;
3173 t3_update_qset_coalesce(q, p);
3174
3175 avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3176 GFP_KERNEL | __GFP_COMP);
3177 if (!avail) {
3178 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3179 ret = -ENOMEM;
3180 goto err;
3181 }
3182 if (avail < q->fl[0].size)
3183 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3184 avail);
3185
3186 avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3187 GFP_KERNEL | __GFP_COMP);
3188 if (avail < q->fl[1].size)
3189 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3190 avail);
3191 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3192
3193 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3194 V_NEWTIMER(q->rspq.holdoff_tmr));
3195
3196 return 0;
3197
3198err_unlock:
3199 spin_unlock_irq(&adapter->sge.reg_lock);
3200err:
3201 t3_free_qset(adapter, q);
3202 return ret;
3203}
3204
3205
3206
3207
3208
3209
3210
3211void t3_start_sge_timers(struct adapter *adap)
3212{
3213 int i;
3214
3215 for (i = 0; i < SGE_QSETS; ++i) {
3216 struct sge_qset *q = &adap->sge.qs[i];
3217
3218 if (q->tx_reclaim_timer.function)
3219 mod_timer(&q->tx_reclaim_timer,
3220 jiffies + TX_RECLAIM_PERIOD);
3221
3222 if (q->rx_reclaim_timer.function)
3223 mod_timer(&q->rx_reclaim_timer,
3224 jiffies + RX_RECLAIM_PERIOD);
3225 }
3226}
3227
3228
3229
3230
3231
3232
3233
3234void t3_stop_sge_timers(struct adapter *adap)
3235{
3236 int i;
3237
3238 for (i = 0; i < SGE_QSETS; ++i) {
3239 struct sge_qset *q = &adap->sge.qs[i];
3240
3241 if (q->tx_reclaim_timer.function)
3242 del_timer_sync(&q->tx_reclaim_timer);
3243 if (q->rx_reclaim_timer.function)
3244 del_timer_sync(&q->rx_reclaim_timer);
3245 }
3246}
3247
3248
3249
3250
3251
3252
3253
3254void t3_free_sge_resources(struct adapter *adap)
3255{
3256 int i;
3257
3258 for (i = 0; i < SGE_QSETS; ++i)
3259 t3_free_qset(adap, &adap->sge.qs[i]);
3260}
3261
3262
3263
3264
3265
3266
3267
3268
3269void t3_sge_start(struct adapter *adap)
3270{
3271 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3272}
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286void t3_sge_stop_dma(struct adapter *adap)
3287{
3288 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3289}
3290
3291
3292
3293
3294
3295
3296
3297
3298void t3_sge_stop(struct adapter *adap)
3299{
3300 int i;
3301
3302 t3_sge_stop_dma(adap);
3303
3304
3305 if (!(adap->flags & FULL_INIT_DONE))
3306 return;
3307 for (i = 0; i < SGE_QSETS; ++i) {
3308 struct sge_qset *qs = &adap->sge.qs[i];
3309
3310 cancel_work_sync(&qs->txq[TXQ_OFLD].qresume_task);
3311 cancel_work_sync(&qs->txq[TXQ_CTRL].qresume_task);
3312 }
3313}
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325void t3_sge_init(struct adapter *adap, struct sge_params *p)
3326{
3327 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3328
3329 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3330 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3331 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3332 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3333#if SGE_NUM_GENBITS == 1
3334 ctrl |= F_EGRGENCTRL;
3335#endif
3336 if (adap->params.rev > 0) {
3337 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3338 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3339 }
3340 t3_write_reg(adap, A_SG_CONTROL, ctrl);
3341 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3342 V_LORCQDRBTHRSH(512));
3343 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3344 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3345 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3346 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3347 adap->params.rev < T3_REV_C ? 1000 : 500);
3348 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3349 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3350 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3351 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3352 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3353}
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3365{
3366 int i;
3367
3368 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3369 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3370
3371 for (i = 0; i < SGE_QSETS; ++i) {
3372 struct qset_params *q = p->qset + i;
3373
3374 q->polling = adap->params.rev > 0;
3375 q->coalesce_usecs = 5;
3376 q->rspq_size = 1024;
3377 q->fl_size = 1024;
3378 q->jumbo_size = 512;
3379 q->txq_size[TXQ_ETH] = 1024;
3380 q->txq_size[TXQ_OFLD] = 1024;
3381 q->txq_size[TXQ_CTRL] = 256;
3382 q->cong_thres = 0;
3383 }
3384
3385 spin_lock_init(&adap->sge.reg_lock);
3386}
3387