1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#include "common.h"
41
42#include <linux/types.h>
43#include <linux/errno.h>
44#include <linux/pci.h>
45#include <linux/ktime.h>
46#include <linux/netdevice.h>
47#include <linux/etherdevice.h>
48#include <linux/if_vlan.h>
49#include <linux/skbuff.h>
50#include <linux/init.h>
51#include <linux/mm.h>
52#include <linux/tcp.h>
53#include <linux/ip.h>
54#include <linux/in.h>
55#include <linux/if_arp.h>
56
57#include "cpl5_cmd.h"
58#include "sge.h"
59#include "regs.h"
60#include "espi.h"
61
62
63#define ETH_P_CPL5 0xf
64
65#define SGE_CMDQ_N 2
66#define SGE_FREELQ_N 2
67#define SGE_CMDQ0_E_N 1024
68#define SGE_CMDQ1_E_N 128
69#define SGE_FREEL_SIZE 4096
70#define SGE_JUMBO_FREEL_SIZE 512
71#define SGE_FREEL_REFILL_THRESH 16
72#define SGE_RESPQ_E_N 1024
73#define SGE_INTRTIMER_NRES 1000
74#define SGE_RX_SM_BUF_SIZE 1536
75#define SGE_TX_DESC_MAX_PLEN 16384
76
77#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)
78
79
80
81
82
83#define TX_RECLAIM_PERIOD (HZ / 4)
84
85#define M_CMD_LEN 0x7fffffff
86#define V_CMD_LEN(v) (v)
87#define G_CMD_LEN(v) ((v) & M_CMD_LEN)
88#define V_CMD_GEN1(v) ((v) << 31)
89#define V_CMD_GEN2(v) (v)
90#define F_CMD_DATAVALID (1 << 1)
91#define F_CMD_SOP (1 << 2)
92#define V_CMD_EOP(v) ((v) << 3)
93
94
95
96
97#if defined(__BIG_ENDIAN_BITFIELD)
98struct cmdQ_e {
99 u32 addr_lo;
100 u32 len_gen;
101 u32 flags;
102 u32 addr_hi;
103};
104
105struct freelQ_e {
106 u32 addr_lo;
107 u32 len_gen;
108 u32 gen2;
109 u32 addr_hi;
110};
111
112struct respQ_e {
113 u32 Qsleeping : 4;
114 u32 Cmdq1CreditReturn : 5;
115 u32 Cmdq1DmaComplete : 5;
116 u32 Cmdq0CreditReturn : 5;
117 u32 Cmdq0DmaComplete : 5;
118 u32 FreelistQid : 2;
119 u32 CreditValid : 1;
120 u32 DataValid : 1;
121 u32 Offload : 1;
122 u32 Eop : 1;
123 u32 Sop : 1;
124 u32 GenerationBit : 1;
125 u32 BufferLength;
126};
127#elif defined(__LITTLE_ENDIAN_BITFIELD)
128struct cmdQ_e {
129 u32 len_gen;
130 u32 addr_lo;
131 u32 addr_hi;
132 u32 flags;
133};
134
135struct freelQ_e {
136 u32 len_gen;
137 u32 addr_lo;
138 u32 addr_hi;
139 u32 gen2;
140};
141
142struct respQ_e {
143 u32 BufferLength;
144 u32 GenerationBit : 1;
145 u32 Sop : 1;
146 u32 Eop : 1;
147 u32 Offload : 1;
148 u32 DataValid : 1;
149 u32 CreditValid : 1;
150 u32 FreelistQid : 2;
151 u32 Cmdq0DmaComplete : 5;
152 u32 Cmdq0CreditReturn : 5;
153 u32 Cmdq1DmaComplete : 5;
154 u32 Cmdq1CreditReturn : 5;
155 u32 Qsleeping : 4;
156} ;
157#endif
158
159
160
161
162struct cmdQ_ce {
163 struct sk_buff *skb;
164 DECLARE_PCI_UNMAP_ADDR(dma_addr);
165 DECLARE_PCI_UNMAP_LEN(dma_len);
166};
167
168struct freelQ_ce {
169 struct sk_buff *skb;
170 DECLARE_PCI_UNMAP_ADDR(dma_addr);
171 DECLARE_PCI_UNMAP_LEN(dma_len);
172};
173
174
175
176
177struct cmdQ {
178 unsigned long status;
179 unsigned int in_use;
180 unsigned int size;
181 unsigned int processed;
182 unsigned int cleaned;
183 unsigned int stop_thres;
184 u16 pidx;
185 u16 cidx;
186 u8 genbit;
187 u8 sop;
188 struct cmdQ_e *entries;
189 struct cmdQ_ce *centries;
190 dma_addr_t dma_addr;
191 spinlock_t lock;
192};
193
194struct freelQ {
195 unsigned int credits;
196 unsigned int size;
197 u16 pidx;
198 u16 cidx;
199 u16 rx_buffer_size;
200 u16 dma_offset;
201 u16 recycleq_idx;
202 u8 genbit;
203 struct freelQ_e *entries;
204 struct freelQ_ce *centries;
205 dma_addr_t dma_addr;
206};
207
208struct respQ {
209 unsigned int credits;
210 unsigned int size;
211 u16 cidx;
212 u8 genbit;
213 struct respQ_e *entries;
214 dma_addr_t dma_addr;
215};
216
217
218enum {
219 CMDQ_STAT_RUNNING = 1,
220 CMDQ_STAT_LAST_PKT_DB = 2
221};
222
223
224
225
226struct sched_port {
227 unsigned int avail;
228 unsigned int drain_bits_per_1024ns;
229 unsigned int speed;
230 unsigned int mtu;
231 struct sk_buff_head skbq;
232};
233
234
235struct sched {
236 ktime_t last_updated;
237 unsigned int max_avail;
238 unsigned int port;
239 unsigned int num;
240 struct sched_port p[MAX_NPORTS];
241 struct tasklet_struct sched_tsk;
242};
243static void restart_sched(unsigned long);
244
245
246
247
248
249
250
251
252
253
254struct sge {
255 struct adapter *adapter;
256 struct net_device *netdev;
257 struct freelQ freelQ[SGE_FREELQ_N];
258 struct respQ respQ;
259 unsigned long stopped_tx_queues;
260 unsigned int rx_pkt_pad;
261 unsigned int jumbo_fl;
262 unsigned int intrtimer_nres;
263 unsigned int fixed_intrtimer;
264 struct timer_list tx_reclaim_timer;
265 struct timer_list espibug_timer;
266 unsigned long espibug_timeout;
267 struct sk_buff *espibug_skb[MAX_NPORTS];
268 u32 sge_control;
269 struct sge_intr_counts stats;
270 struct sge_port_stats *port_stats[MAX_NPORTS];
271 struct sched *tx_sched;
272 struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned_in_smp;
273};
274
275
276
277
278static void tx_sched_stop(struct sge *sge)
279{
280 struct sched *s = sge->tx_sched;
281 int i;
282
283 tasklet_kill(&s->sched_tsk);
284
285 for (i = 0; i < MAX_NPORTS; i++)
286 __skb_queue_purge(&s->p[s->port].skbq);
287}
288
289
290
291
292
293unsigned int t1_sched_update_parms(struct sge *sge, unsigned int port,
294 unsigned int mtu, unsigned int speed)
295{
296 struct sched *s = sge->tx_sched;
297 struct sched_port *p = &s->p[port];
298 unsigned int max_avail_segs;
299
300 pr_debug("t1_sched_update_params mtu=%d speed=%d\n", mtu, speed);
301 if (speed)
302 p->speed = speed;
303 if (mtu)
304 p->mtu = mtu;
305
306 if (speed || mtu) {
307 unsigned long long drain = 1024ULL * p->speed * (p->mtu - 40);
308 do_div(drain, (p->mtu + 50) * 1000);
309 p->drain_bits_per_1024ns = (unsigned int) drain;
310
311 if (p->speed < 1000)
312 p->drain_bits_per_1024ns =
313 90 * p->drain_bits_per_1024ns / 100;
314 }
315
316 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) {
317 p->drain_bits_per_1024ns -= 16;
318 s->max_avail = max(4096U, p->mtu + 16 + 14 + 4);
319 max_avail_segs = max(1U, 4096 / (p->mtu - 40));
320 } else {
321 s->max_avail = 16384;
322 max_avail_segs = max(1U, 9000 / (p->mtu - 40));
323 }
324
325 pr_debug("t1_sched_update_parms: mtu %u speed %u max_avail %u "
326 "max_avail_segs %u drain_bits_per_1024ns %u\n", p->mtu,
327 p->speed, s->max_avail, max_avail_segs,
328 p->drain_bits_per_1024ns);
329
330 return max_avail_segs * (p->mtu - 40);
331}
332
333#if 0
334
335
336
337
338
339void t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val)
340{
341 struct sched *s = sge->tx_sched;
342 unsigned int i;
343
344 s->max_avail = val;
345 for (i = 0; i < MAX_NPORTS; i++)
346 t1_sched_update_parms(sge, i, 0, 0);
347}
348
349
350
351
352
353void t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port,
354 unsigned int val)
355{
356 struct sched *s = sge->tx_sched;
357 struct sched_port *p = &s->p[port];
358 p->drain_bits_per_1024ns = val * 1024 / 1000;
359 t1_sched_update_parms(sge, port, 0, 0);
360}
361
362#endif
363
364
365
366
367
368static inline ktime_t get_clock(void)
369{
370 struct timespec ts;
371
372 ktime_get_ts(&ts);
373 return timespec_to_ktime(ts);
374}
375
376
377
378
379static int tx_sched_init(struct sge *sge)
380{
381 struct sched *s;
382 int i;
383
384 s = kzalloc(sizeof (struct sched), GFP_KERNEL);
385 if (!s)
386 return -ENOMEM;
387
388 pr_debug("tx_sched_init\n");
389 tasklet_init(&s->sched_tsk, restart_sched, (unsigned long) sge);
390 sge->tx_sched = s;
391
392 for (i = 0; i < MAX_NPORTS; i++) {
393 skb_queue_head_init(&s->p[i].skbq);
394 t1_sched_update_parms(sge, i, 1500, 1000);
395 }
396
397 return 0;
398}
399
400
401
402
403
404
405static inline int sched_update_avail(struct sge *sge)
406{
407 struct sched *s = sge->tx_sched;
408 ktime_t now = get_clock();
409 unsigned int i;
410 long long delta_time_ns;
411
412 delta_time_ns = ktime_to_ns(ktime_sub(now, s->last_updated));
413
414 pr_debug("sched_update_avail delta=%lld\n", delta_time_ns);
415 if (delta_time_ns < 15000)
416 return 0;
417
418 for (i = 0; i < MAX_NPORTS; i++) {
419 struct sched_port *p = &s->p[i];
420 unsigned int delta_avail;
421
422 delta_avail = (p->drain_bits_per_1024ns * delta_time_ns) >> 13;
423 p->avail = min(p->avail + delta_avail, s->max_avail);
424 }
425
426 s->last_updated = now;
427
428 return 1;
429}
430
431
432
433
434
435
436
437
438
439static struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb,
440 unsigned int credits)
441{
442 struct sched *s = sge->tx_sched;
443 struct sk_buff_head *skbq;
444 unsigned int i, len, update = 1;
445
446 pr_debug("sched_skb %p\n", skb);
447 if (!skb) {
448 if (!s->num)
449 return NULL;
450 } else {
451 skbq = &s->p[skb->dev->if_port].skbq;
452 __skb_queue_tail(skbq, skb);
453 s->num++;
454 skb = NULL;
455 }
456
457 if (credits < MAX_SKB_FRAGS + 1)
458 goto out;
459
460again:
461 for (i = 0; i < MAX_NPORTS; i++) {
462 s->port = ++s->port & (MAX_NPORTS - 1);
463 skbq = &s->p[s->port].skbq;
464
465 skb = skb_peek(skbq);
466
467 if (!skb)
468 continue;
469
470 len = skb->len;
471 if (len <= s->p[s->port].avail) {
472 s->p[s->port].avail -= len;
473 s->num--;
474 __skb_unlink(skb, skbq);
475 goto out;
476 }
477 skb = NULL;
478 }
479
480 if (update-- && sched_update_avail(sge))
481 goto again;
482
483out:
484
485
486
487 if (s->num && !skb) {
488 struct cmdQ *q = &sge->cmdQ[0];
489 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
490 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
491 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
492 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
493 }
494 }
495 pr_debug("sched_skb ret %p\n", skb);
496
497 return skb;
498}
499
500
501
502
503static inline void doorbell_pio(struct adapter *adapter, u32 val)
504{
505 wmb();
506 writel(val, adapter->regs + A_SG_DOORBELL);
507}
508
509
510
511
512
513static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q)
514{
515 unsigned int cidx = q->cidx;
516
517 while (q->credits--) {
518 struct freelQ_ce *ce = &q->centries[cidx];
519
520 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
521 pci_unmap_len(ce, dma_len),
522 PCI_DMA_FROMDEVICE);
523 dev_kfree_skb(ce->skb);
524 ce->skb = NULL;
525 if (++cidx == q->size)
526 cidx = 0;
527 }
528}
529
530
531
532
533static void free_rx_resources(struct sge *sge)
534{
535 struct pci_dev *pdev = sge->adapter->pdev;
536 unsigned int size, i;
537
538 if (sge->respQ.entries) {
539 size = sizeof(struct respQ_e) * sge->respQ.size;
540 pci_free_consistent(pdev, size, sge->respQ.entries,
541 sge->respQ.dma_addr);
542 }
543
544 for (i = 0; i < SGE_FREELQ_N; i++) {
545 struct freelQ *q = &sge->freelQ[i];
546
547 if (q->centries) {
548 free_freelQ_buffers(pdev, q);
549 kfree(q->centries);
550 }
551 if (q->entries) {
552 size = sizeof(struct freelQ_e) * q->size;
553 pci_free_consistent(pdev, size, q->entries,
554 q->dma_addr);
555 }
556 }
557}
558
559
560
561
562
563static int alloc_rx_resources(struct sge *sge, struct sge_params *p)
564{
565 struct pci_dev *pdev = sge->adapter->pdev;
566 unsigned int size, i;
567
568 for (i = 0; i < SGE_FREELQ_N; i++) {
569 struct freelQ *q = &sge->freelQ[i];
570
571 q->genbit = 1;
572 q->size = p->freelQ_size[i];
573 q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN;
574 size = sizeof(struct freelQ_e) * q->size;
575 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
576 if (!q->entries)
577 goto err_no_mem;
578
579 size = sizeof(struct freelQ_ce) * q->size;
580 q->centries = kzalloc(size, GFP_KERNEL);
581 if (!q->centries)
582 goto err_no_mem;
583 }
584
585
586
587
588
589
590
591
592 sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE +
593 sizeof(struct cpl_rx_data) +
594 sge->freelQ[!sge->jumbo_fl].dma_offset;
595
596 size = (16 * 1024) -
597 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
598
599 sge->freelQ[sge->jumbo_fl].rx_buffer_size = size;
600
601
602
603
604
605 sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0;
606 sge->freelQ[sge->jumbo_fl].recycleq_idx = 1;
607
608 sge->respQ.genbit = 1;
609 sge->respQ.size = SGE_RESPQ_E_N;
610 sge->respQ.credits = 0;
611 size = sizeof(struct respQ_e) * sge->respQ.size;
612 sge->respQ.entries =
613 pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr);
614 if (!sge->respQ.entries)
615 goto err_no_mem;
616 return 0;
617
618err_no_mem:
619 free_rx_resources(sge);
620 return -ENOMEM;
621}
622
623
624
625
626static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n)
627{
628 struct cmdQ_ce *ce;
629 struct pci_dev *pdev = sge->adapter->pdev;
630 unsigned int cidx = q->cidx;
631
632 q->in_use -= n;
633 ce = &q->centries[cidx];
634 while (n--) {
635 if (likely(pci_unmap_len(ce, dma_len))) {
636 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
637 pci_unmap_len(ce, dma_len),
638 PCI_DMA_TODEVICE);
639 if (q->sop)
640 q->sop = 0;
641 }
642 if (ce->skb) {
643 dev_kfree_skb_any(ce->skb);
644 q->sop = 1;
645 }
646 ce++;
647 if (++cidx == q->size) {
648 cidx = 0;
649 ce = q->centries;
650 }
651 }
652 q->cidx = cidx;
653}
654
655
656
657
658
659
660static void free_tx_resources(struct sge *sge)
661{
662 struct pci_dev *pdev = sge->adapter->pdev;
663 unsigned int size, i;
664
665 for (i = 0; i < SGE_CMDQ_N; i++) {
666 struct cmdQ *q = &sge->cmdQ[i];
667
668 if (q->centries) {
669 if (q->in_use)
670 free_cmdQ_buffers(sge, q, q->in_use);
671 kfree(q->centries);
672 }
673 if (q->entries) {
674 size = sizeof(struct cmdQ_e) * q->size;
675 pci_free_consistent(pdev, size, q->entries,
676 q->dma_addr);
677 }
678 }
679}
680
681
682
683
684static int alloc_tx_resources(struct sge *sge, struct sge_params *p)
685{
686 struct pci_dev *pdev = sge->adapter->pdev;
687 unsigned int size, i;
688
689 for (i = 0; i < SGE_CMDQ_N; i++) {
690 struct cmdQ *q = &sge->cmdQ[i];
691
692 q->genbit = 1;
693 q->sop = 1;
694 q->size = p->cmdQ_size[i];
695 q->in_use = 0;
696 q->status = 0;
697 q->processed = q->cleaned = 0;
698 q->stop_thres = 0;
699 spin_lock_init(&q->lock);
700 size = sizeof(struct cmdQ_e) * q->size;
701 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
702 if (!q->entries)
703 goto err_no_mem;
704
705 size = sizeof(struct cmdQ_ce) * q->size;
706 q->centries = kzalloc(size, GFP_KERNEL);
707 if (!q->centries)
708 goto err_no_mem;
709 }
710
711
712
713
714
715
716
717
718 sge->cmdQ[0].stop_thres = sge->adapter->params.nports *
719 (MAX_SKB_FRAGS + 1);
720 return 0;
721
722err_no_mem:
723 free_tx_resources(sge);
724 return -ENOMEM;
725}
726
727static inline void setup_ring_params(struct adapter *adapter, u64 addr,
728 u32 size, int base_reg_lo,
729 int base_reg_hi, int size_reg)
730{
731 writel((u32)addr, adapter->regs + base_reg_lo);
732 writel(addr >> 32, adapter->regs + base_reg_hi);
733 writel(size, adapter->regs + size_reg);
734}
735
736
737
738
739void t1_set_vlan_accel(struct adapter *adapter, int on_off)
740{
741 struct sge *sge = adapter->sge;
742
743 sge->sge_control &= ~F_VLAN_XTRACT;
744 if (on_off)
745 sge->sge_control |= F_VLAN_XTRACT;
746 if (adapter->open_device_map) {
747 writel(sge->sge_control, adapter->regs + A_SG_CONTROL);
748 readl(adapter->regs + A_SG_CONTROL);
749 }
750}
751
752
753
754
755
756static void configure_sge(struct sge *sge, struct sge_params *p)
757{
758 struct adapter *ap = sge->adapter;
759
760 writel(0, ap->regs + A_SG_CONTROL);
761 setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size,
762 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
763 setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size,
764 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
765 setup_ring_params(ap, sge->freelQ[0].dma_addr,
766 sge->freelQ[0].size, A_SG_FL0BASELWR,
767 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
768 setup_ring_params(ap, sge->freelQ[1].dma_addr,
769 sge->freelQ[1].size, A_SG_FL1BASELWR,
770 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
771
772
773 writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD);
774
775 setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size,
776 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
777 writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT);
778
779 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
780 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
781 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
782 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
783
784#if defined(__BIG_ENDIAN_BITFIELD)
785 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
786#endif
787
788
789 sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap);
790
791 t1_sge_set_coalesce_params(sge, p);
792}
793
794
795
796
797static inline unsigned int jumbo_payload_capacity(const struct sge *sge)
798{
799 return sge->freelQ[sge->jumbo_fl].rx_buffer_size -
800 sge->freelQ[sge->jumbo_fl].dma_offset -
801 sizeof(struct cpl_rx_data);
802}
803
804
805
806
807void t1_sge_destroy(struct sge *sge)
808{
809 int i;
810
811 for_each_port(sge->adapter, i)
812 free_percpu(sge->port_stats[i]);
813
814 kfree(sge->tx_sched);
815 free_tx_resources(sge);
816 free_rx_resources(sge);
817 kfree(sge);
818}
819
820
821
822
823
824
825
826
827
828
829
830
831
832static void refill_free_list(struct sge *sge, struct freelQ *q)
833{
834 struct pci_dev *pdev = sge->adapter->pdev;
835 struct freelQ_ce *ce = &q->centries[q->pidx];
836 struct freelQ_e *e = &q->entries[q->pidx];
837 unsigned int dma_len = q->rx_buffer_size - q->dma_offset;
838
839 while (q->credits < q->size) {
840 struct sk_buff *skb;
841 dma_addr_t mapping;
842
843 skb = alloc_skb(q->rx_buffer_size, GFP_ATOMIC);
844 if (!skb)
845 break;
846
847 skb_reserve(skb, q->dma_offset);
848 mapping = pci_map_single(pdev, skb->data, dma_len,
849 PCI_DMA_FROMDEVICE);
850 skb_reserve(skb, sge->rx_pkt_pad);
851
852 ce->skb = skb;
853 pci_unmap_addr_set(ce, dma_addr, mapping);
854 pci_unmap_len_set(ce, dma_len, dma_len);
855 e->addr_lo = (u32)mapping;
856 e->addr_hi = (u64)mapping >> 32;
857 e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit);
858 wmb();
859 e->gen2 = V_CMD_GEN2(q->genbit);
860
861 e++;
862 ce++;
863 if (++q->pidx == q->size) {
864 q->pidx = 0;
865 q->genbit ^= 1;
866 ce = q->centries;
867 e = q->entries;
868 }
869 q->credits++;
870 }
871}
872
873
874
875
876
877
878static void freelQs_empty(struct sge *sge)
879{
880 struct adapter *adapter = sge->adapter;
881 u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE);
882 u32 irqholdoff_reg;
883
884 refill_free_list(sge, &sge->freelQ[0]);
885 refill_free_list(sge, &sge->freelQ[1]);
886
887 if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) &&
888 sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) {
889 irq_reg |= F_FL_EXHAUSTED;
890 irqholdoff_reg = sge->fixed_intrtimer;
891 } else {
892
893 irq_reg &= ~F_FL_EXHAUSTED;
894 irqholdoff_reg = sge->intrtimer_nres;
895 }
896 writel(irqholdoff_reg, adapter->regs + A_SG_INTRTIMER);
897 writel(irq_reg, adapter->regs + A_SG_INT_ENABLE);
898
899
900 doorbell_pio(adapter, F_FL0_ENABLE | F_FL1_ENABLE);
901}
902
903#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
904#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
905#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
906 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
907
908
909
910
911void t1_sge_intr_disable(struct sge *sge)
912{
913 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
914
915 writel(val & ~SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
916 writel(0, sge->adapter->regs + A_SG_INT_ENABLE);
917}
918
919
920
921
922void t1_sge_intr_enable(struct sge *sge)
923{
924 u32 en = SGE_INT_ENABLE;
925 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
926
927 if (sge->adapter->flags & TSO_CAPABLE)
928 en &= ~F_PACKET_TOO_BIG;
929 writel(en, sge->adapter->regs + A_SG_INT_ENABLE);
930 writel(val | SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
931}
932
933
934
935
936void t1_sge_intr_clear(struct sge *sge)
937{
938 writel(SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_CAUSE);
939 writel(0xffffffff, sge->adapter->regs + A_SG_INT_CAUSE);
940}
941
942
943
944
945int t1_sge_intr_error_handler(struct sge *sge)
946{
947 struct adapter *adapter = sge->adapter;
948 u32 cause = readl(adapter->regs + A_SG_INT_CAUSE);
949
950 if (adapter->flags & TSO_CAPABLE)
951 cause &= ~F_PACKET_TOO_BIG;
952 if (cause & F_RESPQ_EXHAUSTED)
953 sge->stats.respQ_empty++;
954 if (cause & F_RESPQ_OVERFLOW) {
955 sge->stats.respQ_overflow++;
956 CH_ALERT("%s: SGE response queue overflow\n",
957 adapter->name);
958 }
959 if (cause & F_FL_EXHAUSTED) {
960 sge->stats.freelistQ_empty++;
961 freelQs_empty(sge);
962 }
963 if (cause & F_PACKET_TOO_BIG) {
964 sge->stats.pkt_too_big++;
965 CH_ALERT("%s: SGE max packet size exceeded\n",
966 adapter->name);
967 }
968 if (cause & F_PACKET_MISMATCH) {
969 sge->stats.pkt_mismatch++;
970 CH_ALERT("%s: SGE packet mismatch\n", adapter->name);
971 }
972 if (cause & SGE_INT_FATAL)
973 t1_fatal_err(adapter);
974
975 writel(cause, adapter->regs + A_SG_INT_CAUSE);
976 return 0;
977}
978
979const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge)
980{
981 return &sge->stats;
982}
983
984void t1_sge_get_port_stats(const struct sge *sge, int port,
985 struct sge_port_stats *ss)
986{
987 int cpu;
988
989 memset(ss, 0, sizeof(*ss));
990 for_each_possible_cpu(cpu) {
991 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[port], cpu);
992
993 ss->rx_cso_good += st->rx_cso_good;
994 ss->tx_cso += st->tx_cso;
995 ss->tx_tso += st->tx_tso;
996 ss->tx_need_hdrroom += st->tx_need_hdrroom;
997 ss->vlan_xtract += st->vlan_xtract;
998 ss->vlan_insert += st->vlan_insert;
999 }
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010static void recycle_fl_buf(struct freelQ *fl, int idx)
1011{
1012 struct freelQ_e *from = &fl->entries[idx];
1013 struct freelQ_e *to = &fl->entries[fl->pidx];
1014
1015 fl->centries[fl->pidx] = fl->centries[idx];
1016 to->addr_lo = from->addr_lo;
1017 to->addr_hi = from->addr_hi;
1018 to->len_gen = G_CMD_LEN(from->len_gen) | V_CMD_GEN1(fl->genbit);
1019 wmb();
1020 to->gen2 = V_CMD_GEN2(fl->genbit);
1021 fl->credits++;
1022
1023 if (++fl->pidx == fl->size) {
1024 fl->pidx = 0;
1025 fl->genbit ^= 1;
1026 }
1027}
1028
1029static int copybreak __read_mostly = 256;
1030module_param(copybreak, int, 0);
1031MODULE_PARM_DESC(copybreak, "Receive copy threshold");
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047static inline struct sk_buff *get_packet(struct pci_dev *pdev,
1048 struct freelQ *fl, unsigned int len)
1049{
1050 struct sk_buff *skb;
1051 const struct freelQ_ce *ce = &fl->centries[fl->cidx];
1052
1053 if (len < copybreak) {
1054 skb = alloc_skb(len + 2, GFP_ATOMIC);
1055 if (!skb)
1056 goto use_orig_buf;
1057
1058 skb_reserve(skb, 2);
1059 skb_put(skb, len);
1060 pci_dma_sync_single_for_cpu(pdev,
1061 pci_unmap_addr(ce, dma_addr),
1062 pci_unmap_len(ce, dma_len),
1063 PCI_DMA_FROMDEVICE);
1064 skb_copy_from_linear_data(ce->skb, skb->data, len);
1065 pci_dma_sync_single_for_device(pdev,
1066 pci_unmap_addr(ce, dma_addr),
1067 pci_unmap_len(ce, dma_len),
1068 PCI_DMA_FROMDEVICE);
1069 recycle_fl_buf(fl, fl->cidx);
1070 return skb;
1071 }
1072
1073use_orig_buf:
1074 if (fl->credits < 2) {
1075 recycle_fl_buf(fl, fl->cidx);
1076 return NULL;
1077 }
1078
1079 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
1080 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1081 skb = ce->skb;
1082 prefetch(skb->data);
1083
1084 skb_put(skb, len);
1085 return skb;
1086}
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097static void unexpected_offload(struct adapter *adapter, struct freelQ *fl)
1098{
1099 struct freelQ_ce *ce = &fl->centries[fl->cidx];
1100 struct sk_buff *skb = ce->skb;
1101
1102 pci_dma_sync_single_for_cpu(adapter->pdev, pci_unmap_addr(ce, dma_addr),
1103 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1104 CH_ERR("%s: unexpected offload packet, cmd %u\n",
1105 adapter->name, *skb->data);
1106 recycle_fl_buf(fl, fl->cidx);
1107}
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119static inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb)
1120{
1121 unsigned int count = 0;
1122
1123 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1124 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
1125 unsigned int i, len = skb->len - skb->data_len;
1126 while (len > SGE_TX_DESC_MAX_PLEN) {
1127 count++;
1128 len -= SGE_TX_DESC_MAX_PLEN;
1129 }
1130 for (i = 0; nfrags--; i++) {
1131 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1132 len = frag->size;
1133 while (len > SGE_TX_DESC_MAX_PLEN) {
1134 count++;
1135 len -= SGE_TX_DESC_MAX_PLEN;
1136 }
1137 }
1138 }
1139 return count;
1140}
1141
1142
1143
1144
1145
1146
1147
1148static inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping,
1149 unsigned int len, unsigned int gen,
1150 unsigned int eop)
1151{
1152 BUG_ON(len > SGE_TX_DESC_MAX_PLEN);
1153
1154 e->addr_lo = (u32)mapping;
1155 e->addr_hi = (u64)mapping >> 32;
1156 e->len_gen = V_CMD_LEN(len) | V_CMD_GEN1(gen);
1157 e->flags = F_CMD_DATAVALID | V_CMD_EOP(eop) | V_CMD_GEN2(gen);
1158}
1159
1160
1161
1162
1163
1164
1165
1166static inline unsigned int write_large_page_tx_descs(unsigned int pidx,
1167 struct cmdQ_e **e,
1168 struct cmdQ_ce **ce,
1169 unsigned int *gen,
1170 dma_addr_t *desc_mapping,
1171 unsigned int *desc_len,
1172 unsigned int nfrags,
1173 struct cmdQ *q)
1174{
1175 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1176 struct cmdQ_e *e1 = *e;
1177 struct cmdQ_ce *ce1 = *ce;
1178
1179 while (*desc_len > SGE_TX_DESC_MAX_PLEN) {
1180 *desc_len -= SGE_TX_DESC_MAX_PLEN;
1181 write_tx_desc(e1, *desc_mapping, SGE_TX_DESC_MAX_PLEN,
1182 *gen, nfrags == 0 && *desc_len == 0);
1183 ce1->skb = NULL;
1184 pci_unmap_len_set(ce1, dma_len, 0);
1185 *desc_mapping += SGE_TX_DESC_MAX_PLEN;
1186 if (*desc_len) {
1187 ce1++;
1188 e1++;
1189 if (++pidx == q->size) {
1190 pidx = 0;
1191 *gen ^= 1;
1192 ce1 = q->centries;
1193 e1 = q->entries;
1194 }
1195 }
1196 }
1197 *e = e1;
1198 *ce = ce1;
1199 }
1200 return pidx;
1201}
1202
1203
1204
1205
1206
1207static inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb,
1208 unsigned int pidx, unsigned int gen,
1209 struct cmdQ *q)
1210{
1211 dma_addr_t mapping, desc_mapping;
1212 struct cmdQ_e *e, *e1;
1213 struct cmdQ_ce *ce;
1214 unsigned int i, flags, first_desc_len, desc_len,
1215 nfrags = skb_shinfo(skb)->nr_frags;
1216
1217 e = e1 = &q->entries[pidx];
1218 ce = &q->centries[pidx];
1219
1220 mapping = pci_map_single(adapter->pdev, skb->data,
1221 skb->len - skb->data_len, PCI_DMA_TODEVICE);
1222
1223 desc_mapping = mapping;
1224 desc_len = skb->len - skb->data_len;
1225
1226 flags = F_CMD_DATAVALID | F_CMD_SOP |
1227 V_CMD_EOP(nfrags == 0 && desc_len <= SGE_TX_DESC_MAX_PLEN) |
1228 V_CMD_GEN2(gen);
1229 first_desc_len = (desc_len <= SGE_TX_DESC_MAX_PLEN) ?
1230 desc_len : SGE_TX_DESC_MAX_PLEN;
1231 e->addr_lo = (u32)desc_mapping;
1232 e->addr_hi = (u64)desc_mapping >> 32;
1233 e->len_gen = V_CMD_LEN(first_desc_len) | V_CMD_GEN1(gen);
1234 ce->skb = NULL;
1235 pci_unmap_len_set(ce, dma_len, 0);
1236
1237 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN &&
1238 desc_len > SGE_TX_DESC_MAX_PLEN) {
1239 desc_mapping += first_desc_len;
1240 desc_len -= first_desc_len;
1241 e1++;
1242 ce++;
1243 if (++pidx == q->size) {
1244 pidx = 0;
1245 gen ^= 1;
1246 e1 = q->entries;
1247 ce = q->centries;
1248 }
1249 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1250 &desc_mapping, &desc_len,
1251 nfrags, q);
1252
1253 if (likely(desc_len))
1254 write_tx_desc(e1, desc_mapping, desc_len, gen,
1255 nfrags == 0);
1256 }
1257
1258 ce->skb = NULL;
1259 pci_unmap_addr_set(ce, dma_addr, mapping);
1260 pci_unmap_len_set(ce, dma_len, skb->len - skb->data_len);
1261
1262 for (i = 0; nfrags--; i++) {
1263 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1264 e1++;
1265 ce++;
1266 if (++pidx == q->size) {
1267 pidx = 0;
1268 gen ^= 1;
1269 e1 = q->entries;
1270 ce = q->centries;
1271 }
1272
1273 mapping = pci_map_page(adapter->pdev, frag->page,
1274 frag->page_offset, frag->size,
1275 PCI_DMA_TODEVICE);
1276 desc_mapping = mapping;
1277 desc_len = frag->size;
1278
1279 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1280 &desc_mapping, &desc_len,
1281 nfrags, q);
1282 if (likely(desc_len))
1283 write_tx_desc(e1, desc_mapping, desc_len, gen,
1284 nfrags == 0);
1285 ce->skb = NULL;
1286 pci_unmap_addr_set(ce, dma_addr, mapping);
1287 pci_unmap_len_set(ce, dma_len, frag->size);
1288 }
1289 ce->skb = skb;
1290 wmb();
1291 e->flags = flags;
1292}
1293
1294
1295
1296
1297static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q)
1298{
1299 unsigned int reclaim = q->processed - q->cleaned;
1300
1301 if (reclaim) {
1302 pr_debug("reclaim_completed_tx processed:%d cleaned:%d\n",
1303 q->processed, q->cleaned);
1304 free_cmdQ_buffers(sge, q, reclaim);
1305 q->cleaned += reclaim;
1306 }
1307}
1308
1309
1310
1311
1312
1313static void restart_sched(unsigned long arg)
1314{
1315 struct sge *sge = (struct sge *) arg;
1316 struct adapter *adapter = sge->adapter;
1317 struct cmdQ *q = &sge->cmdQ[0];
1318 struct sk_buff *skb;
1319 unsigned int credits, queued_skb = 0;
1320
1321 spin_lock(&q->lock);
1322 reclaim_completed_tx(sge, q);
1323
1324 credits = q->size - q->in_use;
1325 pr_debug("restart_sched credits=%d\n", credits);
1326 while ((skb = sched_skb(sge, NULL, credits)) != NULL) {
1327 unsigned int genbit, pidx, count;
1328 count = 1 + skb_shinfo(skb)->nr_frags;
1329 count += compute_large_page_tx_descs(skb);
1330 q->in_use += count;
1331 genbit = q->genbit;
1332 pidx = q->pidx;
1333 q->pidx += count;
1334 if (q->pidx >= q->size) {
1335 q->pidx -= q->size;
1336 q->genbit ^= 1;
1337 }
1338 write_tx_descs(adapter, skb, pidx, genbit, q);
1339 credits = q->size - q->in_use;
1340 queued_skb = 1;
1341 }
1342
1343 if (queued_skb) {
1344 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1345 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1346 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1347 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1348 }
1349 }
1350 spin_unlock(&q->lock);
1351}
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
1362{
1363 struct sk_buff *skb;
1364 const struct cpl_rx_pkt *p;
1365 struct adapter *adapter = sge->adapter;
1366 struct sge_port_stats *st;
1367
1368 skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
1369 if (unlikely(!skb)) {
1370 sge->stats.rx_drops++;
1371 return;
1372 }
1373
1374 p = (const struct cpl_rx_pkt *) skb->data;
1375 if (p->iff >= adapter->params.nports) {
1376 kfree_skb(skb);
1377 return;
1378 }
1379 __skb_pull(skb, sizeof(*p));
1380
1381 st = per_cpu_ptr(sge->port_stats[p->iff], smp_processor_id());
1382
1383 skb->protocol = eth_type_trans(skb, adapter->port[p->iff].dev);
1384 if ((adapter->flags & RX_CSUM_ENABLED) && p->csum == 0xffff &&
1385 skb->protocol == htons(ETH_P_IP) &&
1386 (skb->data[9] == IPPROTO_TCP || skb->data[9] == IPPROTO_UDP)) {
1387 ++st->rx_cso_good;
1388 skb->ip_summed = CHECKSUM_UNNECESSARY;
1389 } else
1390 skb->ip_summed = CHECKSUM_NONE;
1391
1392 if (unlikely(adapter->vlan_grp && p->vlan_valid)) {
1393 st->vlan_xtract++;
1394 vlan_hwaccel_receive_skb(skb, adapter->vlan_grp,
1395 ntohs(p->vlan));
1396 } else
1397 netif_receive_skb(skb);
1398}
1399
1400
1401
1402
1403
1404static inline int enough_free_Tx_descs(const struct cmdQ *q)
1405{
1406 unsigned int r = q->processed - q->cleaned;
1407
1408 return q->in_use - r < (q->size >> 1);
1409}
1410
1411
1412
1413
1414
1415static void restart_tx_queues(struct sge *sge)
1416{
1417 struct adapter *adap = sge->adapter;
1418 int i;
1419
1420 if (!enough_free_Tx_descs(&sge->cmdQ[0]))
1421 return;
1422
1423 for_each_port(adap, i) {
1424 struct net_device *nd = adap->port[i].dev;
1425
1426 if (test_and_clear_bit(nd->if_port, &sge->stopped_tx_queues) &&
1427 netif_running(nd)) {
1428 sge->stats.cmdQ_restarted[2]++;
1429 netif_wake_queue(nd);
1430 }
1431 }
1432}
1433
1434
1435
1436
1437
1438static unsigned int update_tx_info(struct adapter *adapter,
1439 unsigned int flags,
1440 unsigned int pr0)
1441{
1442 struct sge *sge = adapter->sge;
1443 struct cmdQ *cmdq = &sge->cmdQ[0];
1444
1445 cmdq->processed += pr0;
1446 if (flags & (F_FL0_ENABLE | F_FL1_ENABLE)) {
1447 freelQs_empty(sge);
1448 flags &= ~(F_FL0_ENABLE | F_FL1_ENABLE);
1449 }
1450 if (flags & F_CMDQ0_ENABLE) {
1451 clear_bit(CMDQ_STAT_RUNNING, &cmdq->status);
1452
1453 if (cmdq->cleaned + cmdq->in_use != cmdq->processed &&
1454 !test_and_set_bit(CMDQ_STAT_LAST_PKT_DB, &cmdq->status)) {
1455 set_bit(CMDQ_STAT_RUNNING, &cmdq->status);
1456 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1457 }
1458 if (sge->tx_sched)
1459 tasklet_hi_schedule(&sge->tx_sched->sched_tsk);
1460
1461 flags &= ~F_CMDQ0_ENABLE;
1462 }
1463
1464 if (unlikely(sge->stopped_tx_queues != 0))
1465 restart_tx_queues(sge);
1466
1467 return flags;
1468}
1469
1470
1471
1472
1473
1474static int process_responses(struct adapter *adapter, int budget)
1475{
1476 struct sge *sge = adapter->sge;
1477 struct respQ *q = &sge->respQ;
1478 struct respQ_e *e = &q->entries[q->cidx];
1479 int done = 0;
1480 unsigned int flags = 0;
1481 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
1482
1483 while (done < budget && e->GenerationBit == q->genbit) {
1484 flags |= e->Qsleeping;
1485
1486 cmdq_processed[0] += e->Cmdq0CreditReturn;
1487 cmdq_processed[1] += e->Cmdq1CreditReturn;
1488
1489
1490
1491
1492
1493 if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) {
1494 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
1495 cmdq_processed[0] = 0;
1496 }
1497
1498 if (unlikely(cmdq_processed[1] > 16)) {
1499 sge->cmdQ[1].processed += cmdq_processed[1];
1500 cmdq_processed[1] = 0;
1501 }
1502
1503 if (likely(e->DataValid)) {
1504 struct freelQ *fl = &sge->freelQ[e->FreelistQid];
1505
1506 BUG_ON(!e->Sop || !e->Eop);
1507 if (unlikely(e->Offload))
1508 unexpected_offload(adapter, fl);
1509 else
1510 sge_rx(sge, fl, e->BufferLength);
1511
1512 ++done;
1513
1514
1515
1516
1517
1518 if (++fl->cidx == fl->size)
1519 fl->cidx = 0;
1520 prefetch(fl->centries[fl->cidx].skb);
1521
1522 if (unlikely(--fl->credits <
1523 fl->size - SGE_FREEL_REFILL_THRESH))
1524 refill_free_list(sge, fl);
1525 } else
1526 sge->stats.pure_rsps++;
1527
1528 e++;
1529 if (unlikely(++q->cidx == q->size)) {
1530 q->cidx = 0;
1531 q->genbit ^= 1;
1532 e = q->entries;
1533 }
1534 prefetch(e);
1535
1536 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1537 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1538 q->credits = 0;
1539 }
1540 }
1541
1542 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
1543 sge->cmdQ[1].processed += cmdq_processed[1];
1544
1545 return done;
1546}
1547
1548static inline int responses_pending(const struct adapter *adapter)
1549{
1550 const struct respQ *Q = &adapter->sge->respQ;
1551 const struct respQ_e *e = &Q->entries[Q->cidx];
1552
1553 return (e->GenerationBit == Q->genbit);
1554}
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564static int process_pure_responses(struct adapter *adapter)
1565{
1566 struct sge *sge = adapter->sge;
1567 struct respQ *q = &sge->respQ;
1568 struct respQ_e *e = &q->entries[q->cidx];
1569 const struct freelQ *fl = &sge->freelQ[e->FreelistQid];
1570 unsigned int flags = 0;
1571 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
1572
1573 prefetch(fl->centries[fl->cidx].skb);
1574 if (e->DataValid)
1575 return 1;
1576
1577 do {
1578 flags |= e->Qsleeping;
1579
1580 cmdq_processed[0] += e->Cmdq0CreditReturn;
1581 cmdq_processed[1] += e->Cmdq1CreditReturn;
1582
1583 e++;
1584 if (unlikely(++q->cidx == q->size)) {
1585 q->cidx = 0;
1586 q->genbit ^= 1;
1587 e = q->entries;
1588 }
1589 prefetch(e);
1590
1591 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1592 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1593 q->credits = 0;
1594 }
1595 sge->stats.pure_rsps++;
1596 } while (e->GenerationBit == q->genbit && !e->DataValid);
1597
1598 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
1599 sge->cmdQ[1].processed += cmdq_processed[1];
1600
1601 return e->GenerationBit == q->genbit;
1602}
1603
1604
1605
1606
1607
1608
1609int t1_poll(struct napi_struct *napi, int budget)
1610{
1611 struct adapter *adapter = container_of(napi, struct adapter, napi);
1612 int work_done = process_responses(adapter, budget);
1613
1614 if (likely(work_done < budget)) {
1615 napi_complete(napi);
1616 writel(adapter->sge->respQ.cidx,
1617 adapter->regs + A_SG_SLEEPING);
1618 }
1619 return work_done;
1620}
1621
1622irqreturn_t t1_interrupt(int irq, void *data)
1623{
1624 struct adapter *adapter = data;
1625 struct sge *sge = adapter->sge;
1626 int handled;
1627
1628 if (likely(responses_pending(adapter))) {
1629 writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
1630
1631 if (napi_schedule_prep(&adapter->napi)) {
1632 if (process_pure_responses(adapter))
1633 __napi_schedule(&adapter->napi);
1634 else {
1635
1636 writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
1637
1638 napi_enable(&adapter->napi);
1639 }
1640 }
1641 return IRQ_HANDLED;
1642 }
1643
1644 spin_lock(&adapter->async_lock);
1645 handled = t1_slow_intr_handler(adapter);
1646 spin_unlock(&adapter->async_lock);
1647
1648 if (!handled)
1649 sge->stats.unhandled_irqs++;
1650
1651 return IRQ_RETVAL(handled != 0);
1652}
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667static int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter,
1668 unsigned int qid, struct net_device *dev)
1669{
1670 struct sge *sge = adapter->sge;
1671 struct cmdQ *q = &sge->cmdQ[qid];
1672 unsigned int credits, pidx, genbit, count, use_sched_skb = 0;
1673
1674 if (!spin_trylock(&q->lock))
1675 return NETDEV_TX_LOCKED;
1676
1677 reclaim_completed_tx(sge, q);
1678
1679 pidx = q->pidx;
1680 credits = q->size - q->in_use;
1681 count = 1 + skb_shinfo(skb)->nr_frags;
1682 count += compute_large_page_tx_descs(skb);
1683
1684
1685 if (unlikely(credits < count)) {
1686 if (!netif_queue_stopped(dev)) {
1687 netif_stop_queue(dev);
1688 set_bit(dev->if_port, &sge->stopped_tx_queues);
1689 sge->stats.cmdQ_full[2]++;
1690 CH_ERR("%s: Tx ring full while queue awake!\n",
1691 adapter->name);
1692 }
1693 spin_unlock(&q->lock);
1694 return NETDEV_TX_BUSY;
1695 }
1696
1697 if (unlikely(credits - count < q->stop_thres)) {
1698 netif_stop_queue(dev);
1699 set_bit(dev->if_port, &sge->stopped_tx_queues);
1700 sge->stats.cmdQ_full[2]++;
1701 }
1702
1703
1704
1705
1706 if (sge->tx_sched && !qid && skb->dev) {
1707use_sched:
1708 use_sched_skb = 1;
1709
1710
1711
1712 skb = sched_skb(sge, skb, credits);
1713 if (!skb) {
1714 spin_unlock(&q->lock);
1715 return NETDEV_TX_OK;
1716 }
1717 pidx = q->pidx;
1718 count = 1 + skb_shinfo(skb)->nr_frags;
1719 count += compute_large_page_tx_descs(skb);
1720 }
1721
1722 q->in_use += count;
1723 genbit = q->genbit;
1724 pidx = q->pidx;
1725 q->pidx += count;
1726 if (q->pidx >= q->size) {
1727 q->pidx -= q->size;
1728 q->genbit ^= 1;
1729 }
1730 spin_unlock(&q->lock);
1731
1732 write_tx_descs(adapter, skb, pidx, genbit, q);
1733
1734
1735
1736
1737
1738
1739
1740
1741 if (qid)
1742 doorbell_pio(adapter, F_CMDQ1_ENABLE);
1743 else {
1744 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1745 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1746 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1747 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1748 }
1749 }
1750
1751 if (use_sched_skb) {
1752 if (spin_trylock(&q->lock)) {
1753 credits = q->size - q->in_use;
1754 skb = NULL;
1755 goto use_sched;
1756 }
1757 }
1758 return NETDEV_TX_OK;
1759}
1760
1761#define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14))
1762
1763
1764
1765
1766
1767
1768
1769static inline int eth_hdr_len(const void *data)
1770{
1771 const struct ethhdr *e = data;
1772
1773 return e->h_proto == htons(ETH_P_8021Q) ? VLAN_ETH_HLEN : ETH_HLEN;
1774}
1775
1776
1777
1778
1779netdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
1780{
1781 struct adapter *adapter = dev->ml_priv;
1782 struct sge *sge = adapter->sge;
1783 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[dev->if_port],
1784 smp_processor_id());
1785 struct cpl_tx_pkt *cpl;
1786 struct sk_buff *orig_skb = skb;
1787 int ret;
1788
1789 if (skb->protocol == htons(ETH_P_CPL5))
1790 goto send;
1791
1792
1793
1794
1795
1796 if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) {
1797 skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso));
1798 ++st->tx_need_hdrroom;
1799 dev_kfree_skb_any(orig_skb);
1800 if (!skb)
1801 return NETDEV_TX_OK;
1802 }
1803
1804 if (skb_shinfo(skb)->gso_size) {
1805 int eth_type;
1806 struct cpl_tx_pkt_lso *hdr;
1807
1808 ++st->tx_tso;
1809
1810 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1811 CPL_ETH_II : CPL_ETH_II_VLAN;
1812
1813 hdr = (struct cpl_tx_pkt_lso *)skb_push(skb, sizeof(*hdr));
1814 hdr->opcode = CPL_TX_PKT_LSO;
1815 hdr->ip_csum_dis = hdr->l4_csum_dis = 0;
1816 hdr->ip_hdr_words = ip_hdr(skb)->ihl;
1817 hdr->tcp_hdr_words = tcp_hdr(skb)->doff;
1818 hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type,
1819 skb_shinfo(skb)->gso_size));
1820 hdr->len = htonl(skb->len - sizeof(*hdr));
1821 cpl = (struct cpl_tx_pkt *)hdr;
1822 } else {
1823
1824
1825
1826
1827
1828
1829 if (unlikely(skb->len < ETH_HLEN ||
1830 skb->len > dev->mtu + eth_hdr_len(skb->data))) {
1831 pr_debug("%s: packet size %d hdr %d mtu%d\n", dev->name,
1832 skb->len, eth_hdr_len(skb->data), dev->mtu);
1833 dev_kfree_skb_any(skb);
1834 return NETDEV_TX_OK;
1835 }
1836
1837 if (!(adapter->flags & UDP_CSUM_CAPABLE) &&
1838 skb->ip_summed == CHECKSUM_PARTIAL &&
1839 ip_hdr(skb)->protocol == IPPROTO_UDP) {
1840 if (unlikely(skb_checksum_help(skb))) {
1841 pr_debug("%s: unable to do udp checksum\n", dev->name);
1842 dev_kfree_skb_any(skb);
1843 return NETDEV_TX_OK;
1844 }
1845 }
1846
1847
1848
1849
1850 if ((unlikely(!adapter->sge->espibug_skb[dev->if_port]))) {
1851 if (skb->protocol == htons(ETH_P_ARP) &&
1852 arp_hdr(skb)->ar_op == htons(ARPOP_REQUEST)) {
1853 adapter->sge->espibug_skb[dev->if_port] = skb;
1854
1855
1856
1857
1858 skb = skb_get(skb);
1859 }
1860 }
1861
1862 cpl = (struct cpl_tx_pkt *)__skb_push(skb, sizeof(*cpl));
1863 cpl->opcode = CPL_TX_PKT;
1864 cpl->ip_csum_dis = 1;
1865 cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1;
1866
1867
1868 st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL);
1869 }
1870 cpl->iff = dev->if_port;
1871
1872#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
1873 if (adapter->vlan_grp && vlan_tx_tag_present(skb)) {
1874 cpl->vlan_valid = 1;
1875 cpl->vlan = htons(vlan_tx_tag_get(skb));
1876 st->vlan_insert++;
1877 } else
1878#endif
1879 cpl->vlan_valid = 0;
1880
1881send:
1882 ret = t1_sge_tx(skb, adapter, 0, dev);
1883
1884
1885
1886
1887 if (unlikely(ret != NETDEV_TX_OK && skb != orig_skb)) {
1888 dev_kfree_skb_any(skb);
1889 ret = NETDEV_TX_OK;
1890 }
1891 return ret;
1892}
1893
1894
1895
1896
1897static void sge_tx_reclaim_cb(unsigned long data)
1898{
1899 int i;
1900 struct sge *sge = (struct sge *)data;
1901
1902 for (i = 0; i < SGE_CMDQ_N; ++i) {
1903 struct cmdQ *q = &sge->cmdQ[i];
1904
1905 if (!spin_trylock(&q->lock))
1906 continue;
1907
1908 reclaim_completed_tx(sge, q);
1909 if (i == 0 && q->in_use) {
1910 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
1911 }
1912 spin_unlock(&q->lock);
1913 }
1914 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
1915}
1916
1917
1918
1919
1920int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p)
1921{
1922 sge->fixed_intrtimer = p->rx_coalesce_usecs *
1923 core_ticks_per_usec(sge->adapter);
1924 writel(sge->fixed_intrtimer, sge->adapter->regs + A_SG_INTRTIMER);
1925 return 0;
1926}
1927
1928
1929
1930
1931
1932int t1_sge_configure(struct sge *sge, struct sge_params *p)
1933{
1934 if (alloc_rx_resources(sge, p))
1935 return -ENOMEM;
1936 if (alloc_tx_resources(sge, p)) {
1937 free_rx_resources(sge);
1938 return -ENOMEM;
1939 }
1940 configure_sge(sge, p);
1941
1942
1943
1944
1945
1946
1947
1948 p->large_buf_capacity = jumbo_payload_capacity(sge);
1949 return 0;
1950}
1951
1952
1953
1954
1955void t1_sge_stop(struct sge *sge)
1956{
1957 int i;
1958 writel(0, sge->adapter->regs + A_SG_CONTROL);
1959 readl(sge->adapter->regs + A_SG_CONTROL);
1960
1961 if (is_T2(sge->adapter))
1962 del_timer_sync(&sge->espibug_timer);
1963
1964 del_timer_sync(&sge->tx_reclaim_timer);
1965 if (sge->tx_sched)
1966 tx_sched_stop(sge);
1967
1968 for (i = 0; i < MAX_NPORTS; i++)
1969 kfree_skb(sge->espibug_skb[i]);
1970}
1971
1972
1973
1974
1975void t1_sge_start(struct sge *sge)
1976{
1977 refill_free_list(sge, &sge->freelQ[0]);
1978 refill_free_list(sge, &sge->freelQ[1]);
1979
1980 writel(sge->sge_control, sge->adapter->regs + A_SG_CONTROL);
1981 doorbell_pio(sge->adapter, F_FL0_ENABLE | F_FL1_ENABLE);
1982 readl(sge->adapter->regs + A_SG_CONTROL);
1983
1984 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
1985
1986 if (is_T2(sge->adapter))
1987 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
1988}
1989
1990
1991
1992
1993static void espibug_workaround_t204(unsigned long data)
1994{
1995 struct adapter *adapter = (struct adapter *)data;
1996 struct sge *sge = adapter->sge;
1997 unsigned int nports = adapter->params.nports;
1998 u32 seop[MAX_NPORTS];
1999
2000 if (adapter->open_device_map & PORT_MASK) {
2001 int i;
2002
2003 if (t1_espi_get_mon_t204(adapter, &(seop[0]), 0) < 0)
2004 return;
2005
2006 for (i = 0; i < nports; i++) {
2007 struct sk_buff *skb = sge->espibug_skb[i];
2008
2009 if (!netif_running(adapter->port[i].dev) ||
2010 netif_queue_stopped(adapter->port[i].dev) ||
2011 !seop[i] || ((seop[i] & 0xfff) != 0) || !skb)
2012 continue;
2013
2014 if (!skb->cb[0]) {
2015 u8 ch_mac_addr[ETH_ALEN] = {
2016 0x0, 0x7, 0x43, 0x0, 0x0, 0x0
2017 };
2018
2019 skb_copy_to_linear_data_offset(skb,
2020 sizeof(struct cpl_tx_pkt),
2021 ch_mac_addr,
2022 ETH_ALEN);
2023 skb_copy_to_linear_data_offset(skb,
2024 skb->len - 10,
2025 ch_mac_addr,
2026 ETH_ALEN);
2027 skb->cb[0] = 0xff;
2028 }
2029
2030
2031
2032
2033 skb = skb_get(skb);
2034 t1_sge_tx(skb, adapter, 0, adapter->port[i].dev);
2035 }
2036 }
2037 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
2038}
2039
2040static void espibug_workaround(unsigned long data)
2041{
2042 struct adapter *adapter = (struct adapter *)data;
2043 struct sge *sge = adapter->sge;
2044
2045 if (netif_running(adapter->port[0].dev)) {
2046 struct sk_buff *skb = sge->espibug_skb[0];
2047 u32 seop = t1_espi_get_mon(adapter, 0x930, 0);
2048
2049 if ((seop & 0xfff0fff) == 0xfff && skb) {
2050 if (!skb->cb[0]) {
2051 u8 ch_mac_addr[ETH_ALEN] =
2052 {0x0, 0x7, 0x43, 0x0, 0x0, 0x0};
2053 skb_copy_to_linear_data_offset(skb,
2054 sizeof(struct cpl_tx_pkt),
2055 ch_mac_addr,
2056 ETH_ALEN);
2057 skb_copy_to_linear_data_offset(skb,
2058 skb->len - 10,
2059 ch_mac_addr,
2060 ETH_ALEN);
2061 skb->cb[0] = 0xff;
2062 }
2063
2064
2065
2066
2067 skb = skb_get(skb);
2068 t1_sge_tx(skb, adapter, 0, adapter->port[0].dev);
2069 }
2070 }
2071 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
2072}
2073
2074
2075
2076
2077struct sge * __devinit t1_sge_create(struct adapter *adapter,
2078 struct sge_params *p)
2079{
2080 struct sge *sge = kzalloc(sizeof(*sge), GFP_KERNEL);
2081 int i;
2082
2083 if (!sge)
2084 return NULL;
2085
2086 sge->adapter = adapter;
2087 sge->netdev = adapter->port[0].dev;
2088 sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : 2;
2089 sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0;
2090
2091 for_each_port(adapter, i) {
2092 sge->port_stats[i] = alloc_percpu(struct sge_port_stats);
2093 if (!sge->port_stats[i])
2094 goto nomem_port;
2095 }
2096
2097 init_timer(&sge->tx_reclaim_timer);
2098 sge->tx_reclaim_timer.data = (unsigned long)sge;
2099 sge->tx_reclaim_timer.function = sge_tx_reclaim_cb;
2100
2101 if (is_T2(sge->adapter)) {
2102 init_timer(&sge->espibug_timer);
2103
2104 if (adapter->params.nports > 1) {
2105 tx_sched_init(sge);
2106 sge->espibug_timer.function = espibug_workaround_t204;
2107 } else
2108 sge->espibug_timer.function = espibug_workaround;
2109 sge->espibug_timer.data = (unsigned long)sge->adapter;
2110
2111 sge->espibug_timeout = 1;
2112
2113 if (adapter->params.nports > 1)
2114 sge->espibug_timeout = HZ/100;
2115 }
2116
2117
2118 p->cmdQ_size[0] = SGE_CMDQ0_E_N;
2119 p->cmdQ_size[1] = SGE_CMDQ1_E_N;
2120 p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE;
2121 p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE;
2122 if (sge->tx_sched) {
2123 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204)
2124 p->rx_coalesce_usecs = 15;
2125 else
2126 p->rx_coalesce_usecs = 50;
2127 } else
2128 p->rx_coalesce_usecs = 50;
2129
2130 p->coalesce_enable = 0;
2131 p->sample_interval_usecs = 0;
2132
2133 return sge;
2134nomem_port:
2135 while (i >= 0) {
2136 free_percpu(sge->port_stats[i]);
2137 --i;
2138 }
2139 kfree(sge);
2140 return NULL;
2141
2142}
2143