1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/module.h>
28#include <linux/kernel.h>
29#include <linux/netdevice.h>
30#include <linux/init.h>
31#include <linux/etherdevice.h>
32#include <linux/ip.h>
33#include <linux/ratelimit.h>
34#include <linux/string.h>
35#include <linux/interrupt.h>
36#include <net/dst.h>
37#ifdef CONFIG_XFRM
38#include <linux/xfrm.h>
39#include <net/xfrm.h>
40#endif
41
42#include <linux/atomic.h>
43
44#include <asm/octeon/octeon.h>
45
46#include "ethernet-defines.h"
47#include "octeon-ethernet.h"
48#include "ethernet-tx.h"
49#include "ethernet-util.h"
50
51#include <asm/octeon/cvmx-wqe.h>
52#include <asm/octeon/cvmx-fau.h>
53#include <asm/octeon/cvmx-pip.h>
54#include <asm/octeon/cvmx-pko.h>
55#include <asm/octeon/cvmx-helper.h>
56
57#include <asm/octeon/cvmx-gmxx-defs.h>
58
59#define CVM_OCT_SKB_CB(skb) ((u64 *)((skb)->cb))
60
61
62
63
64
65
66
67
68#ifndef GET_SKBUFF_QOS
69#define GET_SKBUFF_QOS(skb) 0
70#endif
71
72static void cvm_oct_tx_do_cleanup(unsigned long arg);
73static DECLARE_TASKLET(cvm_oct_tx_cleanup_tasklet, cvm_oct_tx_do_cleanup, 0);
74
75
76#define MAX_SKB_TO_FREE (MAX_OUT_QUEUE_DEPTH * 2)
77
78static inline int32_t cvm_oct_adjust_skb_to_free(int32_t skb_to_free, int fau)
79{
80 int32_t undo;
81 undo = skb_to_free > 0 ? MAX_SKB_TO_FREE : skb_to_free +
82 MAX_SKB_TO_FREE;
83 if (undo > 0)
84 cvmx_fau_atomic_add32(fau, -undo);
85 skb_to_free = -skb_to_free > MAX_SKB_TO_FREE ? MAX_SKB_TO_FREE :
86 -skb_to_free;
87 return skb_to_free;
88}
89
90static void cvm_oct_kick_tx_poll_watchdog(void)
91{
92 union cvmx_ciu_timx ciu_timx;
93 ciu_timx.u64 = 0;
94 ciu_timx.s.one_shot = 1;
95 ciu_timx.s.len = cvm_oct_tx_poll_interval;
96 cvmx_write_csr(CVMX_CIU_TIMX(1), ciu_timx.u64);
97}
98
99void cvm_oct_free_tx_skbs(struct net_device *dev)
100{
101 int32_t skb_to_free;
102 int qos, queues_per_port;
103 int total_freed = 0;
104 int total_remaining = 0;
105 unsigned long flags;
106 struct octeon_ethernet *priv = netdev_priv(dev);
107
108 queues_per_port = cvmx_pko_get_num_queues(priv->port);
109
110 for (qos = 0; qos < queues_per_port; qos++) {
111 if (skb_queue_len(&priv->tx_free_list[qos]) == 0)
112 continue;
113 skb_to_free = cvmx_fau_fetch_and_add32(priv->fau+qos*4,
114 MAX_SKB_TO_FREE);
115 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free,
116 priv->fau+qos*4);
117
118
119 total_freed += skb_to_free;
120 if (skb_to_free > 0) {
121 struct sk_buff *to_free_list = NULL;
122 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags);
123 while (skb_to_free > 0) {
124 struct sk_buff *t;
125 t = __skb_dequeue(&priv->tx_free_list[qos]);
126 t->next = to_free_list;
127 to_free_list = t;
128 skb_to_free--;
129 }
130 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock,
131 flags);
132
133 while (to_free_list) {
134 struct sk_buff *t = to_free_list;
135 to_free_list = to_free_list->next;
136 dev_kfree_skb_any(t);
137 }
138 }
139 total_remaining += skb_queue_len(&priv->tx_free_list[qos]);
140 }
141 if (total_freed >= 0 && netif_queue_stopped(dev))
142 netif_wake_queue(dev);
143 if (total_remaining)
144 cvm_oct_kick_tx_poll_watchdog();
145}
146
147
148
149
150
151
152
153
154int cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev)
155{
156 cvmx_pko_command_word0_t pko_command;
157 union cvmx_buf_ptr hw_buffer;
158 uint64_t old_scratch;
159 uint64_t old_scratch2;
160 int qos;
161 int i;
162 enum {QUEUE_CORE, QUEUE_HW, QUEUE_DROP} queue_type;
163 struct octeon_ethernet *priv = netdev_priv(dev);
164 struct sk_buff *to_free_list;
165 int32_t skb_to_free;
166 int32_t buffers_to_free;
167 u32 total_to_clean;
168 unsigned long flags;
169#if REUSE_SKBUFFS_WITHOUT_FREE
170 unsigned char *fpa_head;
171#endif
172
173
174
175
176
177 prefetch(priv);
178
179
180
181
182
183
184 if ((CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 > 1) ||
185 (CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 > 1)) {
186 qos = GET_SKBUFF_QOS(skb);
187 if (qos <= 0)
188 qos = 0;
189 else if (qos >= cvmx_pko_get_num_queues(priv->port))
190 qos = 0;
191 } else
192 qos = 0;
193
194 if (USE_ASYNC_IOBDMA) {
195
196 CVMX_SYNCIOBDMA;
197 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
198 old_scratch2 = cvmx_scratch_read64(CVMX_SCR_SCRATCH + 8);
199
200
201
202
203
204 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH + 8,
205 FAU_NUM_PACKET_BUFFERS_TO_FREE,
206 0);
207 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH,
208 priv->fau + qos * 4,
209 MAX_SKB_TO_FREE);
210 }
211
212
213
214
215
216 if (unlikely(skb_shinfo(skb)->nr_frags > 5)) {
217 if (unlikely(__skb_linearize(skb))) {
218 queue_type = QUEUE_DROP;
219 if (USE_ASYNC_IOBDMA) {
220
221
222
223
224 CVMX_SYNCIOBDMA;
225 skb_to_free =
226 cvmx_scratch_read64(CVMX_SCR_SCRATCH);
227 } else {
228
229
230
231
232 skb_to_free = cvmx_fau_fetch_and_add32(
233 priv->fau + qos * 4, MAX_SKB_TO_FREE);
234 }
235 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free,
236 priv->fau + qos * 4);
237 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags);
238 goto skip_xmit;
239 }
240 }
241
242
243
244
245
246
247
248
249
250
251 if ((skb->len < 64) && OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
252 union cvmx_gmxx_prtx_cfg gmx_prt_cfg;
253 int interface = INTERFACE(priv->port);
254 int index = INDEX(priv->port);
255
256 if (interface < 2) {
257
258 gmx_prt_cfg.u64 =
259 cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
260 if (gmx_prt_cfg.s.duplex == 0) {
261 int add_bytes = 64 - skb->len;
262 if ((skb_tail_pointer(skb) + add_bytes) <=
263 skb_end_pointer(skb))
264 memset(__skb_put(skb, add_bytes), 0,
265 add_bytes);
266 }
267 }
268 }
269
270
271 pko_command.u64 = 0;
272 pko_command.s.n2 = 1;
273 pko_command.s.segs = 1;
274 pko_command.s.total_bytes = skb->len;
275 pko_command.s.size0 = CVMX_FAU_OP_SIZE_32;
276 pko_command.s.subone0 = 1;
277
278 pko_command.s.dontfree = 1;
279
280
281 hw_buffer.u64 = 0;
282 if (skb_shinfo(skb)->nr_frags == 0) {
283 hw_buffer.s.addr = XKPHYS_TO_PHYS((u64)skb->data);
284 hw_buffer.s.pool = 0;
285 hw_buffer.s.size = skb->len;
286 } else {
287 hw_buffer.s.addr = XKPHYS_TO_PHYS((u64)skb->data);
288 hw_buffer.s.pool = 0;
289 hw_buffer.s.size = skb_headlen(skb);
290 CVM_OCT_SKB_CB(skb)[0] = hw_buffer.u64;
291 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
292 struct skb_frag_struct *fs = skb_shinfo(skb)->frags + i;
293 hw_buffer.s.addr = XKPHYS_TO_PHYS(
294 (u64)(page_address(fs->page.p) +
295 fs->page_offset));
296 hw_buffer.s.size = fs->size;
297 CVM_OCT_SKB_CB(skb)[i + 1] = hw_buffer.u64;
298 }
299 hw_buffer.s.addr = XKPHYS_TO_PHYS((u64)CVM_OCT_SKB_CB(skb));
300 hw_buffer.s.size = skb_shinfo(skb)->nr_frags + 1;
301 pko_command.s.segs = skb_shinfo(skb)->nr_frags + 1;
302 pko_command.s.gather = 1;
303 goto dont_put_skbuff_in_hw;
304 }
305
306
307
308
309
310
311
312
313
314
315
316#if REUSE_SKBUFFS_WITHOUT_FREE
317 fpa_head = skb->head + 256 - ((unsigned long)skb->head & 0x7f);
318 if (unlikely(skb->data < fpa_head)) {
319
320
321
322
323 goto dont_put_skbuff_in_hw;
324 }
325 if (unlikely
326 ((skb_end_pointer(skb) - fpa_head) < CVMX_FPA_PACKET_POOL_SIZE)) {
327
328
329
330 goto dont_put_skbuff_in_hw;
331 }
332 if (unlikely(skb_shared(skb))) {
333
334
335
336 goto dont_put_skbuff_in_hw;
337 }
338 if (unlikely(skb_cloned(skb))) {
339
340
341
342 goto dont_put_skbuff_in_hw;
343 }
344 if (unlikely(skb_header_cloned(skb))) {
345
346
347
348 goto dont_put_skbuff_in_hw;
349 }
350 if (unlikely(skb->destructor)) {
351
352
353
354 goto dont_put_skbuff_in_hw;
355 }
356 if (unlikely(skb_shinfo(skb)->nr_frags)) {
357
358
359
360 goto dont_put_skbuff_in_hw;
361 }
362 if (unlikely
363 (skb->truesize !=
364 sizeof(*skb) + skb_end_offset(skb))) {
365
366
367
368 goto dont_put_skbuff_in_hw;
369 }
370
371
372
373
374
375 pko_command.s.dontfree = 0;
376
377 hw_buffer.s.back = ((unsigned long)skb->data >> 7) -
378 ((unsigned long)fpa_head >> 7);
379
380 *(struct sk_buff **)(fpa_head - sizeof(void *)) = skb;
381
382
383
384
385
386 dst_release(skb_dst(skb));
387 skb_dst_set(skb, NULL);
388#ifdef CONFIG_XFRM
389 secpath_put(skb->sp);
390 skb->sp = NULL;
391#endif
392 nf_reset(skb);
393
394#ifdef CONFIG_NET_SCHED
395 skb->tc_index = 0;
396#ifdef CONFIG_NET_CLS_ACT
397 skb->tc_verd = 0;
398#endif
399#endif
400#endif
401
402dont_put_skbuff_in_hw:
403
404
405 if (USE_HW_TCPUDP_CHECKSUM && (skb->protocol == htons(ETH_P_IP)) &&
406 (ip_hdr(skb)->version == 4) && (ip_hdr(skb)->ihl == 5) &&
407 ((ip_hdr(skb)->frag_off == 0) || (ip_hdr(skb)->frag_off == 1 << 14))
408 && ((ip_hdr(skb)->protocol == IPPROTO_TCP)
409 || (ip_hdr(skb)->protocol == IPPROTO_UDP))) {
410
411 pko_command.s.ipoffp1 = sizeof(struct ethhdr) + 1;
412 }
413
414 if (USE_ASYNC_IOBDMA) {
415
416 CVMX_SYNCIOBDMA;
417 skb_to_free = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
418 buffers_to_free = cvmx_scratch_read64(CVMX_SCR_SCRATCH + 8);
419 } else {
420
421 skb_to_free = cvmx_fau_fetch_and_add32(priv->fau + qos * 4,
422 MAX_SKB_TO_FREE);
423 buffers_to_free =
424 cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
425 }
426
427 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free, priv->fau+qos*4);
428
429
430
431
432
433 if ((buffers_to_free < -100) && !pko_command.s.dontfree)
434 pko_command.s.dontfree = 1;
435
436 if (pko_command.s.dontfree) {
437 queue_type = QUEUE_CORE;
438 pko_command.s.reg0 = priv->fau+qos*4;
439 } else {
440 queue_type = QUEUE_HW;
441 }
442 if (USE_ASYNC_IOBDMA)
443 cvmx_fau_async_fetch_and_add32(
444 CVMX_SCR_SCRATCH, FAU_TOTAL_TX_TO_CLEAN, 1);
445
446 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags);
447
448
449 if (unlikely(skb_queue_len(&priv->tx_free_list[qos]) >=
450 MAX_OUT_QUEUE_DEPTH)) {
451
452 if (dev->tx_queue_len != 0) {
453
454 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock,
455 flags);
456 netif_stop_queue(dev);
457 spin_lock_irqsave(&priv->tx_free_list[qos].lock,
458 flags);
459 } else {
460
461 queue_type = QUEUE_DROP;
462 goto skip_xmit;
463 }
464 }
465
466 cvmx_pko_send_packet_prepare(priv->port, priv->queue + qos,
467 CVMX_PKO_LOCK_NONE);
468
469
470 if (unlikely(cvmx_pko_send_packet_finish(priv->port,
471 priv->queue + qos,
472 pko_command, hw_buffer,
473 CVMX_PKO_LOCK_NONE))) {
474 printk_ratelimited("%s: Failed to send the packet\n",
475 dev->name);
476 queue_type = QUEUE_DROP;
477 }
478skip_xmit:
479 to_free_list = NULL;
480
481 switch (queue_type) {
482 case QUEUE_DROP:
483 skb->next = to_free_list;
484 to_free_list = skb;
485 priv->stats.tx_dropped++;
486 break;
487 case QUEUE_HW:
488 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, -1);
489 break;
490 case QUEUE_CORE:
491 __skb_queue_tail(&priv->tx_free_list[qos], skb);
492 break;
493 default:
494 BUG();
495 }
496
497 while (skb_to_free > 0) {
498 struct sk_buff *t = __skb_dequeue(&priv->tx_free_list[qos]);
499 t->next = to_free_list;
500 to_free_list = t;
501 skb_to_free--;
502 }
503
504 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, flags);
505
506
507 while (to_free_list) {
508 struct sk_buff *t = to_free_list;
509 to_free_list = to_free_list->next;
510 dev_kfree_skb_any(t);
511 }
512
513 if (USE_ASYNC_IOBDMA) {
514 CVMX_SYNCIOBDMA;
515 total_to_clean = cvmx_scratch_read64(CVMX_SCR_SCRATCH);
516
517 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch);
518 cvmx_scratch_write64(CVMX_SCR_SCRATCH + 8, old_scratch2);
519 } else {
520 total_to_clean = cvmx_fau_fetch_and_add32(
521 FAU_TOTAL_TX_TO_CLEAN, 1);
522 }
523
524 if (total_to_clean & 0x3ff) {
525
526
527
528
529
530
531 tasklet_schedule(&cvm_oct_tx_cleanup_tasklet);
532 }
533
534 cvm_oct_kick_tx_poll_watchdog();
535
536 return NETDEV_TX_OK;
537}
538
539
540
541
542
543
544
545
546int cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev)
547{
548 struct octeon_ethernet *priv = netdev_priv(dev);
549 void *packet_buffer;
550 void *copy_location;
551
552
553 cvmx_wqe_t *work = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL);
554 if (unlikely(work == NULL)) {
555 printk_ratelimited("%s: Failed to allocate a work queue entry\n",
556 dev->name);
557 priv->stats.tx_dropped++;
558 dev_kfree_skb(skb);
559 return 0;
560 }
561
562
563 packet_buffer = cvmx_fpa_alloc(CVMX_FPA_PACKET_POOL);
564 if (unlikely(packet_buffer == NULL)) {
565 printk_ratelimited("%s: Failed to allocate a packet buffer\n",
566 dev->name);
567 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, DONT_WRITEBACK(1));
568 priv->stats.tx_dropped++;
569 dev_kfree_skb(skb);
570 return 0;
571 }
572
573
574
575
576
577
578
579
580
581 copy_location = packet_buffer + sizeof(uint64_t);
582 copy_location += ((CVMX_HELPER_FIRST_MBUFF_SKIP + 7) & 0xfff8) + 6;
583
584
585
586
587
588
589
590 memcpy(copy_location, skb->data, skb->len);
591
592
593
594
595
596 work->hw_chksum = skb->csum;
597 work->len = skb->len;
598 work->ipprt = priv->port;
599 work->qos = priv->port & 0x7;
600 work->grp = pow_send_group;
601 work->tag_type = CVMX_HELPER_INPUT_TAG_TYPE;
602 work->tag = pow_send_group;
603
604 work->word2.u64 = 0;
605 work->word2.s.bufs = 1;
606 work->packet_ptr.u64 = 0;
607 work->packet_ptr.s.addr = cvmx_ptr_to_phys(copy_location);
608 work->packet_ptr.s.pool = CVMX_FPA_PACKET_POOL;
609 work->packet_ptr.s.size = CVMX_FPA_PACKET_POOL_SIZE;
610 work->packet_ptr.s.back = (copy_location - packet_buffer) >> 7;
611
612 if (skb->protocol == htons(ETH_P_IP)) {
613 work->word2.s.ip_offset = 14;
614#if 0
615 work->word2.s.vlan_valid = 0;
616 work->word2.s.vlan_cfi = 0;
617 work->word2.s.vlan_id = 0;
618 work->word2.s.dec_ipcomp = 0;
619#endif
620 work->word2.s.tcp_or_udp =
621 (ip_hdr(skb)->protocol == IPPROTO_TCP)
622 || (ip_hdr(skb)->protocol == IPPROTO_UDP);
623#if 0
624
625 work->word2.s.dec_ipsec = 0;
626
627 work->word2.s.is_v6 = 0;
628
629 work->word2.s.software = 0;
630
631 work->word2.s.L4_error = 0;
632#endif
633 work->word2.s.is_frag = !((ip_hdr(skb)->frag_off == 0)
634 || (ip_hdr(skb)->frag_off ==
635 1 << 14));
636#if 0
637
638 work->word2.s.IP_exc = 0;
639#endif
640 work->word2.s.is_bcast = (skb->pkt_type == PACKET_BROADCAST);
641 work->word2.s.is_mcast = (skb->pkt_type == PACKET_MULTICAST);
642#if 0
643
644 work->word2.s.not_IP = 0;
645
646 work->word2.s.rcv_error = 0;
647
648 work->word2.s.err_code = 0;
649#endif
650
651
652
653
654
655
656 memcpy(work->packet_data, skb->data + 10,
657 sizeof(work->packet_data));
658 } else {
659#if 0
660 work->word2.snoip.vlan_valid = 0;
661 work->word2.snoip.vlan_cfi = 0;
662 work->word2.snoip.vlan_id = 0;
663 work->word2.snoip.software = 0;
664#endif
665 work->word2.snoip.is_rarp = skb->protocol == htons(ETH_P_RARP);
666 work->word2.snoip.is_arp = skb->protocol == htons(ETH_P_ARP);
667 work->word2.snoip.is_bcast =
668 (skb->pkt_type == PACKET_BROADCAST);
669 work->word2.snoip.is_mcast =
670 (skb->pkt_type == PACKET_MULTICAST);
671 work->word2.snoip.not_IP = 1;
672#if 0
673
674 work->word2.snoip.rcv_error = 0;
675
676 work->word2.snoip.err_code = 0;
677#endif
678 memcpy(work->packet_data, skb->data, sizeof(work->packet_data));
679 }
680
681
682 cvmx_pow_work_submit(work, work->tag, work->tag_type, work->qos,
683 work->grp);
684 priv->stats.tx_packets++;
685 priv->stats.tx_bytes += skb->len;
686 dev_kfree_skb(skb);
687 return 0;
688}
689
690
691
692
693
694
695void cvm_oct_tx_shutdown_dev(struct net_device *dev)
696{
697 struct octeon_ethernet *priv = netdev_priv(dev);
698 unsigned long flags;
699 int qos;
700
701 for (qos = 0; qos < 16; qos++) {
702 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags);
703 while (skb_queue_len(&priv->tx_free_list[qos]))
704 dev_kfree_skb_any(__skb_dequeue
705 (&priv->tx_free_list[qos]));
706 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, flags);
707 }
708}
709
710static void cvm_oct_tx_do_cleanup(unsigned long arg)
711{
712 int port;
713
714 for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) {
715 if (cvm_oct_device[port]) {
716 struct net_device *dev = cvm_oct_device[port];
717 cvm_oct_free_tx_skbs(dev);
718 }
719 }
720}
721
722static irqreturn_t cvm_oct_tx_cleanup_watchdog(int cpl, void *dev_id)
723{
724
725 cvmx_write_csr(CVMX_CIU_TIMX(1), 0);
726
727 tasklet_schedule(&cvm_oct_tx_cleanup_tasklet);
728 return IRQ_HANDLED;
729}
730
731void cvm_oct_tx_initialize(void)
732{
733 int i;
734
735
736 cvmx_write_csr(CVMX_CIU_TIMX(1), 0);
737
738 i = request_irq(OCTEON_IRQ_TIMER1,
739 cvm_oct_tx_cleanup_watchdog, 0,
740 "Ethernet", cvm_oct_device);
741
742 if (i)
743 panic("Could not acquire Ethernet IRQ %d\n", OCTEON_IRQ_TIMER1);
744}
745
746void cvm_oct_tx_shutdown(void)
747{
748
749 free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device);
750}
751