1
2
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
7#include <net/ip6_checksum.h>
8
9#include "ionic.h"
10#include "ionic_lif.h"
11#include "ionic_txrx.h"
12
13static void ionic_rx_clean(struct ionic_queue *q,
14 struct ionic_desc_info *desc_info,
15 struct ionic_cq_info *cq_info,
16 void *cb_arg);
17
18static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
19 ionic_desc_cb cb_func, void *cb_arg)
20{
21 DEBUG_STATS_TXQ_POST(q_to_qcq(q), q->head->desc, ring_dbell);
22
23 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
24}
25
26static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
27 ionic_desc_cb cb_func, void *cb_arg)
28{
29 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
30
31 DEBUG_STATS_RX_BUFF_CNT(q_to_qcq(q));
32}
33
34static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
35{
36 return netdev_get_tx_queue(q->lif->netdev, q->index);
37}
38
39static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q,
40 unsigned int len, bool frags)
41{
42 struct ionic_lif *lif = q->lif;
43 struct ionic_rx_stats *stats;
44 struct net_device *netdev;
45 struct sk_buff *skb;
46
47 netdev = lif->netdev;
48 stats = q_to_rx_stats(q);
49
50 if (frags)
51 skb = napi_get_frags(&q_to_qcq(q)->napi);
52 else
53 skb = netdev_alloc_skb_ip_align(netdev, len);
54
55 if (unlikely(!skb)) {
56 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
57 netdev->name, q->name);
58 stats->alloc_err++;
59 return NULL;
60 }
61
62 return skb;
63}
64
65static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
66 struct ionic_desc_info *desc_info,
67 struct ionic_cq_info *cq_info)
68{
69 struct ionic_rxq_comp *comp = cq_info->cq_desc;
70 struct device *dev = q->lif->ionic->dev;
71 struct ionic_page_info *page_info;
72 struct sk_buff *skb;
73 unsigned int i;
74 u16 frag_len;
75 u16 len;
76
77 page_info = &desc_info->pages[0];
78 len = le16_to_cpu(comp->len);
79
80 prefetch(page_address(page_info->page) + NET_IP_ALIGN);
81
82 skb = ionic_rx_skb_alloc(q, len, true);
83 if (unlikely(!skb))
84 return NULL;
85
86 i = comp->num_sg_elems + 1;
87 do {
88 if (unlikely(!page_info->page)) {
89 struct napi_struct *napi = &q_to_qcq(q)->napi;
90
91 napi->skb = NULL;
92 dev_kfree_skb(skb);
93 return NULL;
94 }
95
96 frag_len = min(len, (u16)PAGE_SIZE);
97 len -= frag_len;
98
99 dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr),
100 PAGE_SIZE, DMA_FROM_DEVICE);
101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
102 page_info->page, 0, frag_len, PAGE_SIZE);
103 page_info->page = NULL;
104 page_info++;
105 i--;
106 } while (i > 0);
107
108 return skb;
109}
110
111static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
112 struct ionic_desc_info *desc_info,
113 struct ionic_cq_info *cq_info)
114{
115 struct ionic_rxq_comp *comp = cq_info->cq_desc;
116 struct device *dev = q->lif->ionic->dev;
117 struct ionic_page_info *page_info;
118 struct sk_buff *skb;
119 u16 len;
120
121 page_info = &desc_info->pages[0];
122 len = le16_to_cpu(comp->len);
123
124 skb = ionic_rx_skb_alloc(q, len, false);
125 if (unlikely(!skb))
126 return NULL;
127
128 if (unlikely(!page_info->page)) {
129 dev_kfree_skb(skb);
130 return NULL;
131 }
132
133 dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr),
134 len, DMA_FROM_DEVICE);
135 skb_copy_to_linear_data(skb, page_address(page_info->page), len);
136 dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr),
137 len, DMA_FROM_DEVICE);
138
139 skb_put(skb, len);
140 skb->protocol = eth_type_trans(skb, q->lif->netdev);
141
142 return skb;
143}
144
145static void ionic_rx_clean(struct ionic_queue *q,
146 struct ionic_desc_info *desc_info,
147 struct ionic_cq_info *cq_info,
148 void *cb_arg)
149{
150 struct ionic_rxq_comp *comp = cq_info->cq_desc;
151 struct ionic_qcq *qcq = q_to_qcq(q);
152 struct ionic_rx_stats *stats;
153 struct net_device *netdev;
154 struct sk_buff *skb;
155
156 stats = q_to_rx_stats(q);
157 netdev = q->lif->netdev;
158
159 if (comp->status) {
160 stats->dropped++;
161 return;
162 }
163
164 stats->pkts++;
165 stats->bytes += le16_to_cpu(comp->len);
166
167 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
168 skb = ionic_rx_copybreak(q, desc_info, cq_info);
169 else
170 skb = ionic_rx_frags(q, desc_info, cq_info);
171
172 if (unlikely(!skb)) {
173 stats->dropped++;
174 return;
175 }
176
177 skb_record_rx_queue(skb, q->index);
178
179 if (likely(netdev->features & NETIF_F_RXHASH)) {
180 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
181 case IONIC_PKT_TYPE_IPV4:
182 case IONIC_PKT_TYPE_IPV6:
183 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
184 PKT_HASH_TYPE_L3);
185 break;
186 case IONIC_PKT_TYPE_IPV4_TCP:
187 case IONIC_PKT_TYPE_IPV6_TCP:
188 case IONIC_PKT_TYPE_IPV4_UDP:
189 case IONIC_PKT_TYPE_IPV6_UDP:
190 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
191 PKT_HASH_TYPE_L4);
192 break;
193 }
194 }
195
196 if (likely(netdev->features & NETIF_F_RXCSUM)) {
197 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
198 skb->ip_summed = CHECKSUM_COMPLETE;
199 skb->csum = (__wsum)le16_to_cpu(comp->csum);
200 stats->csum_complete++;
201 }
202 } else {
203 stats->csum_none++;
204 }
205
206 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
207 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
208 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
209 stats->csum_error++;
210
211 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
212 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
213 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
214 le16_to_cpu(comp->vlan_tci));
215 stats->vlan_stripped++;
216 }
217
218 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
219 napi_gro_receive(&qcq->napi, skb);
220 else
221 napi_gro_frags(&qcq->napi);
222}
223
224static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
225{
226 struct ionic_rxq_comp *comp = cq_info->cq_desc;
227 struct ionic_queue *q = cq->bound_q;
228 struct ionic_desc_info *desc_info;
229
230 if (!color_match(comp->pkt_type_color, cq->done_color))
231 return false;
232
233
234 if (q->tail->index == q->head->index)
235 return false;
236
237 desc_info = q->tail;
238 if (desc_info->index != le16_to_cpu(comp->comp_index))
239 return false;
240
241 q->tail = desc_info->next;
242
243
244 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
245
246 desc_info->cb = NULL;
247 desc_info->cb_arg = NULL;
248
249 return true;
250}
251
252static u32 ionic_rx_walk_cq(struct ionic_cq *rxcq, u32 limit)
253{
254 u32 work_done = 0;
255
256 while (ionic_rx_service(rxcq, rxcq->tail)) {
257 if (rxcq->tail->last)
258 rxcq->done_color = !rxcq->done_color;
259 rxcq->tail = rxcq->tail->next;
260 DEBUG_STATS_CQE_CNT(rxcq);
261
262 if (++work_done >= limit)
263 break;
264 }
265
266 return work_done;
267}
268
269void ionic_rx_flush(struct ionic_cq *cq)
270{
271 struct ionic_dev *idev = &cq->lif->ionic->idev;
272 u32 work_done;
273
274 work_done = ionic_rx_walk_cq(cq, cq->num_descs);
275
276 if (work_done)
277 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
278 work_done, IONIC_INTR_CRED_RESET_COALESCE);
279}
280
281static struct page *ionic_rx_page_alloc(struct ionic_queue *q,
282 dma_addr_t *dma_addr)
283{
284 struct ionic_lif *lif = q->lif;
285 struct ionic_rx_stats *stats;
286 struct net_device *netdev;
287 struct device *dev;
288 struct page *page;
289
290 netdev = lif->netdev;
291 dev = lif->ionic->dev;
292 stats = q_to_rx_stats(q);
293 page = alloc_page(GFP_ATOMIC);
294 if (unlikely(!page)) {
295 net_err_ratelimited("%s: Page alloc failed on %s!\n",
296 netdev->name, q->name);
297 stats->alloc_err++;
298 return NULL;
299 }
300
301 *dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
302 if (unlikely(dma_mapping_error(dev, *dma_addr))) {
303 __free_page(page);
304 net_err_ratelimited("%s: DMA single map failed on %s!\n",
305 netdev->name, q->name);
306 stats->dma_map_err++;
307 return NULL;
308 }
309
310 return page;
311}
312
313static void ionic_rx_page_free(struct ionic_queue *q, struct page *page,
314 dma_addr_t dma_addr)
315{
316 struct ionic_lif *lif = q->lif;
317 struct net_device *netdev;
318 struct device *dev;
319
320 netdev = lif->netdev;
321 dev = lif->ionic->dev;
322
323 if (unlikely(!page)) {
324 net_err_ratelimited("%s: Trying to free unallocated buffer on %s!\n",
325 netdev->name, q->name);
326 return;
327 }
328
329 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
330
331 __free_page(page);
332}
333
334#define IONIC_RX_RING_DOORBELL_STRIDE ((1 << 5) - 1)
335#define IONIC_RX_RING_HEAD_BUF_SZ 2048
336
337void ionic_rx_fill(struct ionic_queue *q)
338{
339 struct net_device *netdev = q->lif->netdev;
340 struct ionic_desc_info *desc_info;
341 struct ionic_page_info *page_info;
342 struct ionic_rxq_sg_desc *sg_desc;
343 struct ionic_rxq_sg_elem *sg_elem;
344 struct ionic_rxq_desc *desc;
345 unsigned int remain_len;
346 unsigned int seg_len;
347 unsigned int nfrags;
348 bool ring_doorbell;
349 unsigned int i, j;
350 unsigned int len;
351
352 len = netdev->mtu + ETH_HLEN;
353 nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE;
354
355 for (i = ionic_q_space_avail(q); i; i--) {
356 remain_len = len;
357 desc_info = q->head;
358 desc = desc_info->desc;
359 sg_desc = desc_info->sg_desc;
360 page_info = &desc_info->pages[0];
361
362 if (page_info->page) {
363 ring_doorbell = ((q->head->index + 1) &
364 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
365 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
366 continue;
367 }
368
369
370 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
371 IONIC_RXQ_DESC_OPCODE_SIMPLE;
372 desc_info->npages = nfrags;
373 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
374 if (unlikely(!page_info->page)) {
375 desc->addr = 0;
376 desc->len = 0;
377 return;
378 }
379 desc->addr = cpu_to_le64(page_info->dma_addr);
380 seg_len = min_t(unsigned int, PAGE_SIZE, len);
381 desc->len = cpu_to_le16(seg_len);
382 remain_len -= seg_len;
383 page_info++;
384
385
386 for (j = 0; j < nfrags - 1; j++) {
387 if (page_info->page)
388 continue;
389
390 sg_elem = &sg_desc->elems[j];
391 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
392 if (unlikely(!page_info->page)) {
393 sg_elem->addr = 0;
394 sg_elem->len = 0;
395 return;
396 }
397 sg_elem->addr = cpu_to_le64(page_info->dma_addr);
398 seg_len = min_t(unsigned int, PAGE_SIZE, remain_len);
399 sg_elem->len = cpu_to_le16(seg_len);
400 remain_len -= seg_len;
401 page_info++;
402 }
403
404 ring_doorbell = ((q->head->index + 1) &
405 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
406 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
407 }
408}
409
410static void ionic_rx_fill_cb(void *arg)
411{
412 ionic_rx_fill(arg);
413}
414
415void ionic_rx_empty(struct ionic_queue *q)
416{
417 struct ionic_desc_info *cur;
418 struct ionic_rxq_desc *desc;
419 unsigned int i;
420
421 for (cur = q->tail; cur != q->head; cur = cur->next) {
422 desc = cur->desc;
423 desc->addr = 0;
424 desc->len = 0;
425
426 for (i = 0; i < cur->npages; i++) {
427 if (likely(cur->pages[i].page)) {
428 ionic_rx_page_free(q, cur->pages[i].page,
429 cur->pages[i].dma_addr);
430 cur->pages[i].page = NULL;
431 cur->pages[i].dma_addr = 0;
432 }
433 }
434
435 cur->cb_arg = NULL;
436 }
437}
438
439int ionic_rx_napi(struct napi_struct *napi, int budget)
440{
441 struct ionic_qcq *qcq = napi_to_qcq(napi);
442 struct ionic_cq *rxcq = napi_to_cq(napi);
443 unsigned int qi = rxcq->bound_q->index;
444 struct ionic_dev *idev;
445 struct ionic_lif *lif;
446 struct ionic_cq *txcq;
447 u32 work_done = 0;
448 u32 flags = 0;
449
450 lif = rxcq->bound_q->lif;
451 idev = &lif->ionic->idev;
452 txcq = &lif->txqcqs[qi].qcq->cq;
453
454 ionic_tx_flush(txcq);
455
456 work_done = ionic_rx_walk_cq(rxcq, budget);
457
458 if (work_done)
459 ionic_rx_fill_cb(rxcq->bound_q);
460
461 if (work_done < budget && napi_complete_done(napi, work_done)) {
462 flags |= IONIC_INTR_CRED_UNMASK;
463 DEBUG_STATS_INTR_REARM(rxcq->bound_intr);
464 }
465
466 if (work_done || flags) {
467 flags |= IONIC_INTR_CRED_RESET_COALESCE;
468 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
469 work_done, flags);
470 }
471
472 DEBUG_STATS_NAPI_POLL(qcq, work_done);
473
474 return work_done;
475}
476
477static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
478 void *data, size_t len)
479{
480 struct ionic_tx_stats *stats = q_to_tx_stats(q);
481 struct device *dev = q->lif->ionic->dev;
482 dma_addr_t dma_addr;
483
484 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
485 if (dma_mapping_error(dev, dma_addr)) {
486 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
487 q->lif->netdev->name, q->name);
488 stats->dma_map_err++;
489 return 0;
490 }
491 return dma_addr;
492}
493
494static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
495 const skb_frag_t *frag,
496 size_t offset, size_t len)
497{
498 struct ionic_tx_stats *stats = q_to_tx_stats(q);
499 struct device *dev = q->lif->ionic->dev;
500 dma_addr_t dma_addr;
501
502 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
503 if (dma_mapping_error(dev, dma_addr)) {
504 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
505 q->lif->netdev->name, q->name);
506 stats->dma_map_err++;
507 }
508 return dma_addr;
509}
510
511static void ionic_tx_clean(struct ionic_queue *q,
512 struct ionic_desc_info *desc_info,
513 struct ionic_cq_info *cq_info,
514 void *cb_arg)
515{
516 struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc;
517 struct ionic_txq_sg_elem *elem = sg_desc->elems;
518 struct ionic_tx_stats *stats = q_to_tx_stats(q);
519 struct ionic_txq_desc *desc = desc_info->desc;
520 struct device *dev = q->lif->ionic->dev;
521 u8 opcode, flags, nsge;
522 u16 queue_index;
523 unsigned int i;
524 u64 addr;
525
526 decode_txq_desc_cmd(le64_to_cpu(desc->cmd),
527 &opcode, &flags, &nsge, &addr);
528
529
530
531
532 if (opcode != IONIC_TXQ_DESC_OPCODE_TSO ||
533 flags & IONIC_TXQ_DESC_FLAG_TSO_SOT)
534 dma_unmap_single(dev, (dma_addr_t)addr,
535 le16_to_cpu(desc->len), DMA_TO_DEVICE);
536 else
537 dma_unmap_page(dev, (dma_addr_t)addr,
538 le16_to_cpu(desc->len), DMA_TO_DEVICE);
539
540 for (i = 0; i < nsge; i++, elem++)
541 dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr),
542 le16_to_cpu(elem->len), DMA_TO_DEVICE);
543
544 if (cb_arg) {
545 struct sk_buff *skb = cb_arg;
546 u32 len = skb->len;
547
548 queue_index = skb_get_queue_mapping(skb);
549 if (unlikely(__netif_subqueue_stopped(q->lif->netdev,
550 queue_index))) {
551 netif_wake_subqueue(q->lif->netdev, queue_index);
552 q->wake++;
553 }
554 dev_kfree_skb_any(skb);
555 stats->clean++;
556 netdev_tx_completed_queue(q_to_ndq(q), 1, len);
557 }
558}
559
560void ionic_tx_flush(struct ionic_cq *cq)
561{
562 struct ionic_txq_comp *comp = cq->tail->cq_desc;
563 struct ionic_dev *idev = &cq->lif->ionic->idev;
564 struct ionic_queue *q = cq->bound_q;
565 struct ionic_desc_info *desc_info;
566 unsigned int work_done = 0;
567
568
569 while (work_done < cq->num_descs &&
570 color_match(comp->color, cq->done_color)) {
571
572
573
574
575 do {
576 desc_info = q->tail;
577 q->tail = desc_info->next;
578 ionic_tx_clean(q, desc_info, cq->tail,
579 desc_info->cb_arg);
580 desc_info->cb = NULL;
581 desc_info->cb_arg = NULL;
582 } while (desc_info->index != le16_to_cpu(comp->comp_index));
583
584 if (cq->tail->last)
585 cq->done_color = !cq->done_color;
586
587 cq->tail = cq->tail->next;
588 comp = cq->tail->cq_desc;
589 DEBUG_STATS_CQE_CNT(cq);
590
591 work_done++;
592 }
593
594 if (work_done)
595 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
596 work_done, 0);
597}
598
599void ionic_tx_empty(struct ionic_queue *q)
600{
601 struct ionic_desc_info *desc_info;
602 int done = 0;
603
604
605 while (q->head != q->tail) {
606 desc_info = q->tail;
607 q->tail = desc_info->next;
608 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
609 desc_info->cb = NULL;
610 desc_info->cb_arg = NULL;
611 done++;
612 }
613}
614
615static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
616{
617 int err;
618
619 err = skb_cow_head(skb, 0);
620 if (err)
621 return err;
622
623 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
624 inner_ip_hdr(skb)->check = 0;
625 inner_tcp_hdr(skb)->check =
626 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
627 inner_ip_hdr(skb)->daddr,
628 0, IPPROTO_TCP, 0);
629 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
630 inner_tcp_hdr(skb)->check =
631 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
632 &inner_ipv6_hdr(skb)->daddr,
633 0, IPPROTO_TCP, 0);
634 }
635
636 return 0;
637}
638
639static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
640{
641 int err;
642
643 err = skb_cow_head(skb, 0);
644 if (err)
645 return err;
646
647 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
648 ip_hdr(skb)->check = 0;
649 tcp_hdr(skb)->check =
650 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
651 ip_hdr(skb)->daddr,
652 0, IPPROTO_TCP, 0);
653 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
654 tcp_v6_gso_csum_prep(skb);
655 }
656
657 return 0;
658}
659
660static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
661 struct sk_buff *skb,
662 dma_addr_t addr, u8 nsge, u16 len,
663 unsigned int hdrlen, unsigned int mss,
664 bool outer_csum,
665 u16 vlan_tci, bool has_vlan,
666 bool start, bool done)
667{
668 u8 flags = 0;
669 u64 cmd;
670
671 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
672 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
673 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
674 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
675
676 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
677 desc->cmd = cpu_to_le64(cmd);
678 desc->len = cpu_to_le16(len);
679 desc->vlan_tci = cpu_to_le16(vlan_tci);
680 desc->hdr_len = cpu_to_le16(hdrlen);
681 desc->mss = cpu_to_le16(mss);
682
683 if (done) {
684 skb_tx_timestamp(skb);
685 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
686 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
687 } else {
688 ionic_txq_post(q, false, ionic_tx_clean, NULL);
689 }
690}
691
692static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q,
693 struct ionic_txq_sg_elem **elem)
694{
695 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
696 struct ionic_txq_desc *desc = q->head->desc;
697
698 *elem = sg_desc->elems;
699 return desc;
700}
701
702static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
703{
704 struct ionic_tx_stats *stats = q_to_tx_stats(q);
705 struct ionic_desc_info *abort = q->head;
706 struct device *dev = q->lif->ionic->dev;
707 struct ionic_desc_info *rewind = abort;
708 struct ionic_txq_sg_elem *elem;
709 struct ionic_txq_desc *desc;
710 unsigned int frag_left = 0;
711 unsigned int offset = 0;
712 unsigned int len_left;
713 dma_addr_t desc_addr;
714 unsigned int hdrlen;
715 unsigned int nfrags;
716 unsigned int seglen;
717 u64 total_bytes = 0;
718 u64 total_pkts = 0;
719 unsigned int left;
720 unsigned int len;
721 unsigned int mss;
722 skb_frag_t *frag;
723 bool start, done;
724 bool outer_csum;
725 bool has_vlan;
726 u16 desc_len;
727 u8 desc_nsge;
728 u16 vlan_tci;
729 bool encap;
730 int err;
731
732 mss = skb_shinfo(skb)->gso_size;
733 nfrags = skb_shinfo(skb)->nr_frags;
734 len_left = skb->len - skb_headlen(skb);
735 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
736 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
737 has_vlan = !!skb_vlan_tag_present(skb);
738 vlan_tci = skb_vlan_tag_get(skb);
739 encap = skb->encapsulation;
740
741
742
743
744
745
746 if (encap)
747 err = ionic_tx_tcp_inner_pseudo_csum(skb);
748 else
749 err = ionic_tx_tcp_pseudo_csum(skb);
750 if (err)
751 return err;
752
753 if (encap)
754 hdrlen = skb_inner_transport_header(skb) - skb->data +
755 inner_tcp_hdrlen(skb);
756 else
757 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
758
759 seglen = hdrlen + mss;
760 left = skb_headlen(skb);
761
762 desc = ionic_tx_tso_next(q, &elem);
763 start = true;
764
765
766
767 while (left > 0) {
768 len = min(seglen, left);
769 frag_left = seglen - len;
770 desc_addr = ionic_tx_map_single(q, skb->data + offset, len);
771 if (dma_mapping_error(dev, desc_addr))
772 goto err_out_abort;
773 desc_len = len;
774 desc_nsge = 0;
775 left -= len;
776 offset += len;
777 if (nfrags > 0 && frag_left > 0)
778 continue;
779 done = (nfrags == 0 && left == 0);
780 ionic_tx_tso_post(q, desc, skb,
781 desc_addr, desc_nsge, desc_len,
782 hdrlen, mss,
783 outer_csum,
784 vlan_tci, has_vlan,
785 start, done);
786 total_pkts++;
787 total_bytes += start ? len : len + hdrlen;
788 desc = ionic_tx_tso_next(q, &elem);
789 start = false;
790 seglen = mss;
791 }
792
793
794
795 for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
796 offset = 0;
797 left = skb_frag_size(frag);
798 len_left -= left;
799 nfrags--;
800 stats->frags++;
801
802 while (left > 0) {
803 if (frag_left > 0) {
804 len = min(frag_left, left);
805 frag_left -= len;
806 elem->addr =
807 cpu_to_le64(ionic_tx_map_frag(q, frag,
808 offset, len));
809 if (dma_mapping_error(dev, elem->addr))
810 goto err_out_abort;
811 elem->len = cpu_to_le16(len);
812 elem++;
813 desc_nsge++;
814 left -= len;
815 offset += len;
816 if (nfrags > 0 && frag_left > 0)
817 continue;
818 done = (nfrags == 0 && left == 0);
819 ionic_tx_tso_post(q, desc, skb, desc_addr,
820 desc_nsge, desc_len,
821 hdrlen, mss, outer_csum,
822 vlan_tci, has_vlan,
823 start, done);
824 total_pkts++;
825 total_bytes += start ? len : len + hdrlen;
826 desc = ionic_tx_tso_next(q, &elem);
827 start = false;
828 } else {
829 len = min(mss, left);
830 frag_left = mss - len;
831 desc_addr = ionic_tx_map_frag(q, frag,
832 offset, len);
833 if (dma_mapping_error(dev, desc_addr))
834 goto err_out_abort;
835 desc_len = len;
836 desc_nsge = 0;
837 left -= len;
838 offset += len;
839 if (nfrags > 0 && frag_left > 0)
840 continue;
841 done = (nfrags == 0 && left == 0);
842 ionic_tx_tso_post(q, desc, skb, desc_addr,
843 desc_nsge, desc_len,
844 hdrlen, mss, outer_csum,
845 vlan_tci, has_vlan,
846 start, done);
847 total_pkts++;
848 total_bytes += start ? len : len + hdrlen;
849 desc = ionic_tx_tso_next(q, &elem);
850 start = false;
851 }
852 }
853 }
854
855 stats->pkts += total_pkts;
856 stats->bytes += total_bytes;
857 stats->tso++;
858 stats->tso_bytes += total_bytes;
859
860 return 0;
861
862err_out_abort:
863 while (rewind->desc != q->head->desc) {
864 ionic_tx_clean(q, rewind, NULL, NULL);
865 rewind = rewind->next;
866 }
867 q->head = abort;
868
869 return -ENOMEM;
870}
871
872static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb)
873{
874 struct ionic_tx_stats *stats = q_to_tx_stats(q);
875 struct ionic_txq_desc *desc = q->head->desc;
876 struct device *dev = q->lif->ionic->dev;
877 dma_addr_t dma_addr;
878 bool has_vlan;
879 u8 flags = 0;
880 bool encap;
881 u64 cmd;
882
883 has_vlan = !!skb_vlan_tag_present(skb);
884 encap = skb->encapsulation;
885
886 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
887 if (dma_mapping_error(dev, dma_addr))
888 return -ENOMEM;
889
890 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
891 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
892
893 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
894 flags, skb_shinfo(skb)->nr_frags, dma_addr);
895 desc->cmd = cpu_to_le64(cmd);
896 desc->len = cpu_to_le16(skb_headlen(skb));
897 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
898 desc->csum_offset = cpu_to_le16(skb->csum_offset);
899 if (has_vlan) {
900 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
901 stats->vlan_inserted++;
902 }
903
904 if (skb->csum_not_inet)
905 stats->crc32_csum++;
906 else
907 stats->csum++;
908
909 return 0;
910}
911
912static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb)
913{
914 struct ionic_tx_stats *stats = q_to_tx_stats(q);
915 struct ionic_txq_desc *desc = q->head->desc;
916 struct device *dev = q->lif->ionic->dev;
917 dma_addr_t dma_addr;
918 bool has_vlan;
919 u8 flags = 0;
920 bool encap;
921 u64 cmd;
922
923 has_vlan = !!skb_vlan_tag_present(skb);
924 encap = skb->encapsulation;
925
926 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
927 if (dma_mapping_error(dev, dma_addr))
928 return -ENOMEM;
929
930 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
931 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
932
933 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
934 flags, skb_shinfo(skb)->nr_frags, dma_addr);
935 desc->cmd = cpu_to_le64(cmd);
936 desc->len = cpu_to_le16(skb_headlen(skb));
937 if (has_vlan) {
938 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
939 stats->vlan_inserted++;
940 }
941
942 stats->csum_none++;
943
944 return 0;
945}
946
947static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb)
948{
949 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
950 unsigned int len_left = skb->len - skb_headlen(skb);
951 struct ionic_txq_sg_elem *elem = sg_desc->elems;
952 struct ionic_tx_stats *stats = q_to_tx_stats(q);
953 struct device *dev = q->lif->ionic->dev;
954 dma_addr_t dma_addr;
955 skb_frag_t *frag;
956 u16 len;
957
958 for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) {
959 len = skb_frag_size(frag);
960 elem->len = cpu_to_le16(len);
961 dma_addr = ionic_tx_map_frag(q, frag, 0, len);
962 if (dma_mapping_error(dev, dma_addr))
963 return -ENOMEM;
964 elem->addr = cpu_to_le64(dma_addr);
965 len_left -= len;
966 stats->frags++;
967 }
968
969 return 0;
970}
971
972static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
973{
974 struct ionic_tx_stats *stats = q_to_tx_stats(q);
975 int err;
976
977
978 if (skb->ip_summed == CHECKSUM_PARTIAL)
979 err = ionic_tx_calc_csum(q, skb);
980 else
981 err = ionic_tx_calc_no_csum(q, skb);
982 if (err)
983 return err;
984
985
986 err = ionic_tx_skb_frags(q, skb);
987 if (err)
988 return err;
989
990 skb_tx_timestamp(skb);
991 stats->pkts++;
992 stats->bytes += skb->len;
993
994 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
995 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
996
997 return 0;
998}
999
1000static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1001{
1002 int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems;
1003 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1004 int err;
1005
1006
1007 if (skb_is_gso(skb))
1008 return (skb->len / skb_shinfo(skb)->gso_size) + 1;
1009
1010
1011 if (skb_shinfo(skb)->nr_frags <= sg_elems)
1012 return 1;
1013
1014
1015 err = skb_linearize(skb);
1016 if (err)
1017 return err;
1018
1019 stats->linearize++;
1020
1021
1022 return 1;
1023}
1024
1025static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1026{
1027 int stopped = 0;
1028
1029 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1030 netif_stop_subqueue(q->lif->netdev, q->index);
1031 q->stop++;
1032 stopped = 1;
1033
1034
1035 smp_rmb();
1036 if (ionic_q_has_space(q, ndescs)) {
1037 netif_wake_subqueue(q->lif->netdev, q->index);
1038 stopped = 0;
1039 }
1040 }
1041
1042 return stopped;
1043}
1044
1045netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1046{
1047 u16 queue_index = skb_get_queue_mapping(skb);
1048 struct ionic_lif *lif = netdev_priv(netdev);
1049 struct ionic_queue *q;
1050 int ndescs;
1051 int err;
1052
1053 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1054 dev_kfree_skb(skb);
1055 return NETDEV_TX_OK;
1056 }
1057
1058 if (unlikely(!lif_to_txqcq(lif, queue_index)))
1059 queue_index = 0;
1060 q = lif_to_txq(lif, queue_index);
1061
1062 ndescs = ionic_tx_descs_needed(q, skb);
1063 if (ndescs < 0)
1064 goto err_out_drop;
1065
1066 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1067 return NETDEV_TX_BUSY;
1068
1069 if (skb_is_gso(skb))
1070 err = ionic_tx_tso(q, skb);
1071 else
1072 err = ionic_tx(q, skb);
1073
1074 if (err)
1075 goto err_out_drop;
1076
1077
1078
1079
1080
1081 ionic_maybe_stop_tx(q, 4);
1082
1083 return NETDEV_TX_OK;
1084
1085err_out_drop:
1086 q->stop++;
1087 q->drop++;
1088 dev_kfree_skb(skb);
1089 return NETDEV_TX_OK;
1090}
1091
