1
2
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
7#include <net/ip6_checksum.h>
8
9#include "ionic.h"
10#include "ionic_lif.h"
11#include "ionic_txrx.h"
12
13
14static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
15 ionic_desc_cb cb_func, void *cb_arg)
16{
17 DEBUG_STATS_TXQ_POST(q, ring_dbell);
18
19 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
20}
21
22static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
23 ionic_desc_cb cb_func, void *cb_arg)
24{
25 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
26
27 DEBUG_STATS_RX_BUFF_CNT(q);
28}
29
30static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
31{
32 return netdev_get_tx_queue(q->lif->netdev, q->index);
33}
34
35static int ionic_rx_page_alloc(struct ionic_queue *q,
36 struct ionic_buf_info *buf_info)
37{
38 struct net_device *netdev = q->lif->netdev;
39 struct ionic_rx_stats *stats;
40 struct device *dev;
41 struct page *page;
42
43 dev = q->dev;
44 stats = q_to_rx_stats(q);
45
46 if (unlikely(!buf_info)) {
47 net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
48 netdev->name, q->name);
49 return -EINVAL;
50 }
51
52 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
53 if (unlikely(!page)) {
54 net_err_ratelimited("%s: %s page alloc failed\n",
55 netdev->name, q->name);
56 stats->alloc_err++;
57 return -ENOMEM;
58 }
59
60 buf_info->dma_addr = dma_map_page(dev, page, 0,
61 IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
62 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
63 __free_pages(page, 0);
64 net_err_ratelimited("%s: %s dma map failed\n",
65 netdev->name, q->name);
66 stats->dma_map_err++;
67 return -EIO;
68 }
69
70 buf_info->page = page;
71 buf_info->page_offset = 0;
72
73 return 0;
74}
75
76static void ionic_rx_page_free(struct ionic_queue *q,
77 struct ionic_buf_info *buf_info)
78{
79 struct net_device *netdev = q->lif->netdev;
80 struct device *dev = q->dev;
81
82 if (unlikely(!buf_info)) {
83 net_err_ratelimited("%s: %s invalid buf_info in free\n",
84 netdev->name, q->name);
85 return;
86 }
87
88 if (!buf_info->page)
89 return;
90
91 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
92 __free_pages(buf_info->page, 0);
93 buf_info->page = NULL;
94}
95
96static bool ionic_rx_buf_recycle(struct ionic_queue *q,
97 struct ionic_buf_info *buf_info, u32 used)
98{
99 u32 size;
100
101
102 if (page_is_pfmemalloc(buf_info->page))
103 return false;
104
105
106 if (page_to_nid(buf_info->page) != numa_mem_id())
107 return false;
108
109 size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
110 buf_info->page_offset += size;
111 if (buf_info->page_offset >= IONIC_PAGE_SIZE)
112 return false;
113
114 get_page(buf_info->page);
115
116 return true;
117}
118
119static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
120 struct ionic_desc_info *desc_info,
121 struct ionic_rxq_comp *comp)
122{
123 struct net_device *netdev = q->lif->netdev;
124 struct ionic_buf_info *buf_info;
125 struct ionic_rx_stats *stats;
126 struct device *dev = q->dev;
127 struct sk_buff *skb;
128 unsigned int i;
129 u16 frag_len;
130 u16 len;
131
132 stats = q_to_rx_stats(q);
133
134 buf_info = &desc_info->bufs[0];
135 len = le16_to_cpu(comp->len);
136
137 prefetchw(buf_info->page);
138
139 skb = napi_get_frags(&q_to_qcq(q)->napi);
140 if (unlikely(!skb)) {
141 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
142 netdev->name, q->name);
143 stats->alloc_err++;
144 return NULL;
145 }
146
147 i = comp->num_sg_elems + 1;
148 do {
149 if (unlikely(!buf_info->page)) {
150 dev_kfree_skb(skb);
151 return NULL;
152 }
153
154 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset);
155 len -= frag_len;
156
157 dma_sync_single_for_cpu(dev,
158 buf_info->dma_addr + buf_info->page_offset,
159 frag_len, DMA_FROM_DEVICE);
160
161 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
162 buf_info->page, buf_info->page_offset, frag_len,
163 IONIC_PAGE_SIZE);
164
165 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
166 dma_unmap_page(dev, buf_info->dma_addr,
167 IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
168 buf_info->page = NULL;
169 }
170
171 buf_info++;
172
173 i--;
174 } while (i > 0);
175
176 return skb;
177}
178
179static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
180 struct ionic_desc_info *desc_info,
181 struct ionic_rxq_comp *comp)
182{
183 struct net_device *netdev = q->lif->netdev;
184 struct ionic_buf_info *buf_info;
185 struct ionic_rx_stats *stats;
186 struct device *dev = q->dev;
187 struct sk_buff *skb;
188 u16 len;
189
190 stats = q_to_rx_stats(q);
191
192 buf_info = &desc_info->bufs[0];
193 len = le16_to_cpu(comp->len);
194
195 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
196 if (unlikely(!skb)) {
197 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
198 netdev->name, q->name);
199 stats->alloc_err++;
200 return NULL;
201 }
202
203 if (unlikely(!buf_info->page)) {
204 dev_kfree_skb(skb);
205 return NULL;
206 }
207
208 dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
209 len, DMA_FROM_DEVICE);
210 skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
211 dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
212 len, DMA_FROM_DEVICE);
213
214 skb_put(skb, len);
215 skb->protocol = eth_type_trans(skb, q->lif->netdev);
216
217 return skb;
218}
219
220static void ionic_rx_clean(struct ionic_queue *q,
221 struct ionic_desc_info *desc_info,
222 struct ionic_cq_info *cq_info,
223 void *cb_arg)
224{
225 struct net_device *netdev = q->lif->netdev;
226 struct ionic_qcq *qcq = q_to_qcq(q);
227 struct ionic_rx_stats *stats;
228 struct ionic_rxq_comp *comp;
229 struct sk_buff *skb;
230
231 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
232
233 stats = q_to_rx_stats(q);
234
235 if (comp->status) {
236 stats->dropped++;
237 return;
238 }
239
240 stats->pkts++;
241 stats->bytes += le16_to_cpu(comp->len);
242
243 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
244 skb = ionic_rx_copybreak(q, desc_info, comp);
245 else
246 skb = ionic_rx_frags(q, desc_info, comp);
247
248 if (unlikely(!skb)) {
249 stats->dropped++;
250 return;
251 }
252
253 skb_record_rx_queue(skb, q->index);
254
255 if (likely(netdev->features & NETIF_F_RXHASH)) {
256 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
257 case IONIC_PKT_TYPE_IPV4:
258 case IONIC_PKT_TYPE_IPV6:
259 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
260 PKT_HASH_TYPE_L3);
261 break;
262 case IONIC_PKT_TYPE_IPV4_TCP:
263 case IONIC_PKT_TYPE_IPV6_TCP:
264 case IONIC_PKT_TYPE_IPV4_UDP:
265 case IONIC_PKT_TYPE_IPV6_UDP:
266 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
267 PKT_HASH_TYPE_L4);
268 break;
269 }
270 }
271
272 if (likely(netdev->features & NETIF_F_RXCSUM) &&
273 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
274 skb->ip_summed = CHECKSUM_COMPLETE;
275 skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
276 stats->csum_complete++;
277 } else {
278 stats->csum_none++;
279 }
280
281 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
282 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
283 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
284 stats->csum_error++;
285
286 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
287 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
288 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
289 le16_to_cpu(comp->vlan_tci));
290 stats->vlan_stripped++;
291 }
292
293 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
294 __le64 *cq_desc_hwstamp;
295 u64 hwstamp;
296
297 cq_desc_hwstamp =
298 cq_info->cq_desc +
299 qcq->cq.desc_size -
300 sizeof(struct ionic_rxq_comp) -
301 IONIC_HWSTAMP_CQ_NEGOFFSET;
302
303 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
304
305 if (hwstamp != IONIC_HWSTAMP_INVALID) {
306 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
307 stats->hwstamp_valid++;
308 } else {
309 stats->hwstamp_invalid++;
310 }
311 }
312
313 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
314 napi_gro_receive(&qcq->napi, skb);
315 else
316 napi_gro_frags(&qcq->napi);
317}
318
319bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
320{
321 struct ionic_queue *q = cq->bound_q;
322 struct ionic_desc_info *desc_info;
323 struct ionic_rxq_comp *comp;
324
325 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
326
327 if (!color_match(comp->pkt_type_color, cq->done_color))
328 return false;
329
330
331 if (q->tail_idx == q->head_idx)
332 return false;
333
334 if (q->tail_idx != le16_to_cpu(comp->comp_index))
335 return false;
336
337 desc_info = &q->info[q->tail_idx];
338 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
339
340
341 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
342
343 desc_info->cb = NULL;
344 desc_info->cb_arg = NULL;
345
346 return true;
347}
348
349void ionic_rx_fill(struct ionic_queue *q)
350{
351 struct net_device *netdev = q->lif->netdev;
352 struct ionic_desc_info *desc_info;
353 struct ionic_rxq_sg_desc *sg_desc;
354 struct ionic_rxq_sg_elem *sg_elem;
355 struct ionic_buf_info *buf_info;
356 struct ionic_rxq_desc *desc;
357 unsigned int remain_len;
358 unsigned int frag_len;
359 unsigned int nfrags;
360 unsigned int i, j;
361 unsigned int len;
362
363 len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
364
365 for (i = ionic_q_space_avail(q); i; i--) {
366 nfrags = 0;
367 remain_len = len;
368 desc_info = &q->info[q->head_idx];
369 desc = desc_info->desc;
370 buf_info = &desc_info->bufs[0];
371
372 if (!buf_info->page) {
373 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
374 desc->addr = 0;
375 desc->len = 0;
376 return;
377 }
378 }
379
380
381 desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
382 frag_len = min_t(u16, len, IONIC_PAGE_SIZE - buf_info->page_offset);
383 desc->len = cpu_to_le16(frag_len);
384 remain_len -= frag_len;
385 buf_info++;
386 nfrags++;
387
388
389 sg_desc = desc_info->sg_desc;
390 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
391 sg_elem = &sg_desc->elems[j];
392 if (!buf_info->page) {
393 if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
394 sg_elem->addr = 0;
395 sg_elem->len = 0;
396 return;
397 }
398 }
399
400 sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
401 frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE - buf_info->page_offset);
402 sg_elem->len = cpu_to_le16(frag_len);
403 remain_len -= frag_len;
404 buf_info++;
405 nfrags++;
406 }
407
408
409 if (j < q->max_sg_elems) {
410 sg_elem = &sg_desc->elems[j];
411 memset(sg_elem, 0, sizeof(*sg_elem));
412 }
413
414 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
415 IONIC_RXQ_DESC_OPCODE_SIMPLE;
416 desc_info->nbufs = nfrags;
417
418 ionic_rxq_post(q, false, ionic_rx_clean, NULL);
419 }
420
421 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
422 q->dbval | q->head_idx);
423}
424
425void ionic_rx_empty(struct ionic_queue *q)
426{
427 struct ionic_desc_info *desc_info;
428 struct ionic_buf_info *buf_info;
429 unsigned int i, j;
430
431 for (i = 0; i < q->num_descs; i++) {
432 desc_info = &q->info[i];
433 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
434 buf_info = &desc_info->bufs[j];
435 if (buf_info->page)
436 ionic_rx_page_free(q, buf_info);
437 }
438
439 desc_info->nbufs = 0;
440 desc_info->cb = NULL;
441 desc_info->cb_arg = NULL;
442 }
443
444 q->head_idx = 0;
445 q->tail_idx = 0;
446}
447
448static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
449{
450 struct dim_sample dim_sample;
451 struct ionic_lif *lif;
452 unsigned int qi;
453 u64 pkts, bytes;
454
455 if (!qcq->intr.dim_coal_hw)
456 return;
457
458 lif = qcq->q.lif;
459 qi = qcq->cq.bound_q->index;
460
461 switch (napi_mode) {
462 case IONIC_LIF_F_TX_DIM_INTR:
463 pkts = lif->txqstats[qi].pkts;
464 bytes = lif->txqstats[qi].bytes;
465 break;
466 case IONIC_LIF_F_RX_DIM_INTR:
467 pkts = lif->rxqstats[qi].pkts;
468 bytes = lif->rxqstats[qi].bytes;
469 break;
470 default:
471 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
472 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
473 break;
474 }
475
476 dim_update_sample(qcq->cq.bound_intr->rearm_count,
477 pkts, bytes, &dim_sample);
478
479 net_dim(&qcq->dim, dim_sample);
480}
481
482int ionic_tx_napi(struct napi_struct *napi, int budget)
483{
484 struct ionic_qcq *qcq = napi_to_qcq(napi);
485 struct ionic_cq *cq = napi_to_cq(napi);
486 struct ionic_dev *idev;
487 struct ionic_lif *lif;
488 u32 work_done = 0;
489 u32 flags = 0;
490
491 lif = cq->bound_q->lif;
492 idev = &lif->ionic->idev;
493
494 work_done = ionic_cq_service(cq, budget,
495 ionic_tx_service, NULL, NULL);
496
497 if (work_done < budget && napi_complete_done(napi, work_done)) {
498 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
499 flags |= IONIC_INTR_CRED_UNMASK;
500 cq->bound_intr->rearm_count++;
501 }
502
503 if (work_done || flags) {
504 flags |= IONIC_INTR_CRED_RESET_COALESCE;
505 ionic_intr_credits(idev->intr_ctrl,
506 cq->bound_intr->index,
507 work_done, flags);
508 }
509
510 DEBUG_STATS_NAPI_POLL(qcq, work_done);
511
512 return work_done;
513}
514
515int ionic_rx_napi(struct napi_struct *napi, int budget)
516{
517 struct ionic_qcq *qcq = napi_to_qcq(napi);
518 struct ionic_cq *cq = napi_to_cq(napi);
519 struct ionic_dev *idev;
520 struct ionic_lif *lif;
521 u16 rx_fill_threshold;
522 u32 work_done = 0;
523 u32 flags = 0;
524
525 lif = cq->bound_q->lif;
526 idev = &lif->ionic->idev;
527
528 work_done = ionic_cq_service(cq, budget,
529 ionic_rx_service, NULL, NULL);
530
531 rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,
532 cq->num_descs / IONIC_RX_FILL_DIV);
533 if (work_done && ionic_q_space_avail(cq->bound_q) >= rx_fill_threshold)
534 ionic_rx_fill(cq->bound_q);
535
536 if (work_done < budget && napi_complete_done(napi, work_done)) {
537 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
538 flags |= IONIC_INTR_CRED_UNMASK;
539 cq->bound_intr->rearm_count++;
540 }
541
542 if (work_done || flags) {
543 flags |= IONIC_INTR_CRED_RESET_COALESCE;
544 ionic_intr_credits(idev->intr_ctrl,
545 cq->bound_intr->index,
546 work_done, flags);
547 }
548
549 DEBUG_STATS_NAPI_POLL(qcq, work_done);
550
551 return work_done;
552}
553
554int ionic_txrx_napi(struct napi_struct *napi, int budget)
555{
556 struct ionic_qcq *qcq = napi_to_qcq(napi);
557 struct ionic_cq *rxcq = napi_to_cq(napi);
558 unsigned int qi = rxcq->bound_q->index;
559 struct ionic_dev *idev;
560 struct ionic_lif *lif;
561 struct ionic_cq *txcq;
562 u16 rx_fill_threshold;
563 u32 rx_work_done = 0;
564 u32 tx_work_done = 0;
565 u32 flags = 0;
566
567 lif = rxcq->bound_q->lif;
568 idev = &lif->ionic->idev;
569 txcq = &lif->txqcqs[qi]->cq;
570
571 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT,
572 ionic_tx_service, NULL, NULL);
573
574 rx_work_done = ionic_cq_service(rxcq, budget,
575 ionic_rx_service, NULL, NULL);
576
577 rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,
578 rxcq->num_descs / IONIC_RX_FILL_DIV);
579 if (rx_work_done && ionic_q_space_avail(rxcq->bound_q) >= rx_fill_threshold)
580 ionic_rx_fill(rxcq->bound_q);
581
582 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
583 ionic_dim_update(qcq, 0);
584 flags |= IONIC_INTR_CRED_UNMASK;
585 rxcq->bound_intr->rearm_count++;
586 }
587
588 if (rx_work_done || flags) {
589 flags |= IONIC_INTR_CRED_RESET_COALESCE;
590 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
591 tx_work_done + rx_work_done, flags);
592 }
593
594 DEBUG_STATS_NAPI_POLL(qcq, rx_work_done);
595 DEBUG_STATS_NAPI_POLL(qcq, tx_work_done);
596
597 return rx_work_done;
598}
599
600static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
601 void *data, size_t len)
602{
603 struct ionic_tx_stats *stats = q_to_tx_stats(q);
604 struct device *dev = q->dev;
605 dma_addr_t dma_addr;
606
607 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
608 if (dma_mapping_error(dev, dma_addr)) {
609 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
610 q->lif->netdev->name, q->name);
611 stats->dma_map_err++;
612 return 0;
613 }
614 return dma_addr;
615}
616
617static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
618 const skb_frag_t *frag,
619 size_t offset, size_t len)
620{
621 struct ionic_tx_stats *stats = q_to_tx_stats(q);
622 struct device *dev = q->dev;
623 dma_addr_t dma_addr;
624
625 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
626 if (dma_mapping_error(dev, dma_addr)) {
627 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
628 q->lif->netdev->name, q->name);
629 stats->dma_map_err++;
630 }
631 return dma_addr;
632}
633
634static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
635 struct ionic_desc_info *desc_info)
636{
637 struct ionic_buf_info *buf_info = desc_info->bufs;
638 struct ionic_tx_stats *stats = q_to_tx_stats(q);
639 struct device *dev = q->dev;
640 dma_addr_t dma_addr;
641 unsigned int nfrags;
642 skb_frag_t *frag;
643 int frag_idx;
644
645 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
646 if (dma_mapping_error(dev, dma_addr)) {
647 stats->dma_map_err++;
648 return -EIO;
649 }
650 buf_info->dma_addr = dma_addr;
651 buf_info->len = skb_headlen(skb);
652 buf_info++;
653
654 frag = skb_shinfo(skb)->frags;
655 nfrags = skb_shinfo(skb)->nr_frags;
656 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
657 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
658 if (dma_mapping_error(dev, dma_addr)) {
659 stats->dma_map_err++;
660 goto dma_fail;
661 }
662 buf_info->dma_addr = dma_addr;
663 buf_info->len = skb_frag_size(frag);
664 buf_info++;
665 }
666
667 desc_info->nbufs = 1 + nfrags;
668
669 return 0;
670
671dma_fail:
672
673 while (frag_idx > 0) {
674 frag_idx--;
675 buf_info--;
676 dma_unmap_page(dev, buf_info->dma_addr,
677 buf_info->len, DMA_TO_DEVICE);
678 }
679 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
680 return -EIO;
681}
682
683static void ionic_tx_clean(struct ionic_queue *q,
684 struct ionic_desc_info *desc_info,
685 struct ionic_cq_info *cq_info,
686 void *cb_arg)
687{
688 struct ionic_buf_info *buf_info = desc_info->bufs;
689 struct ionic_tx_stats *stats = q_to_tx_stats(q);
690 struct ionic_qcq *qcq = q_to_qcq(q);
691 struct sk_buff *skb = cb_arg;
692 struct device *dev = q->dev;
693 unsigned int i;
694 u16 qi;
695
696 if (desc_info->nbufs) {
697 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
698 buf_info->len, DMA_TO_DEVICE);
699 buf_info++;
700 for (i = 1; i < desc_info->nbufs; i++, buf_info++)
701 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
702 buf_info->len, DMA_TO_DEVICE);
703 }
704
705 if (!skb)
706 return;
707
708 qi = skb_get_queue_mapping(skb);
709
710 if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) {
711 if (cq_info) {
712 struct skb_shared_hwtstamps hwts = {};
713 __le64 *cq_desc_hwstamp;
714 u64 hwstamp;
715
716 cq_desc_hwstamp =
717 cq_info->cq_desc +
718 qcq->cq.desc_size -
719 sizeof(struct ionic_txq_comp) -
720 IONIC_HWSTAMP_CQ_NEGOFFSET;
721
722 hwstamp = le64_to_cpu(*cq_desc_hwstamp);
723
724 if (hwstamp != IONIC_HWSTAMP_INVALID) {
725 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
726
727 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
728 skb_tstamp_tx(skb, &hwts);
729
730 stats->hwstamp_valid++;
731 } else {
732 stats->hwstamp_invalid++;
733 }
734 }
735
736 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {
737 netif_wake_subqueue(q->lif->netdev, qi);
738 q->wake++;
739 }
740
741 desc_info->bytes = skb->len;
742 stats->clean++;
743
744 dev_consume_skb_any(skb);
745}
746
747bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
748{
749 struct ionic_queue *q = cq->bound_q;
750 struct ionic_desc_info *desc_info;
751 struct ionic_txq_comp *comp;
752 int bytes = 0;
753 int pkts = 0;
754 u16 index;
755
756 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
757
758 if (!color_match(comp->color, cq->done_color))
759 return false;
760
761
762
763
764 do {
765 desc_info = &q->info[q->tail_idx];
766 desc_info->bytes = 0;
767 index = q->tail_idx;
768 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
769 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
770 if (desc_info->cb_arg) {
771 pkts++;
772 bytes += desc_info->bytes;
773 }
774 desc_info->cb = NULL;
775 desc_info->cb_arg = NULL;
776 } while (index != le16_to_cpu(comp->comp_index));
777
778 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
779 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
780
781 return true;
782}
783
784void ionic_tx_flush(struct ionic_cq *cq)
785{
786 struct ionic_dev *idev = &cq->lif->ionic->idev;
787 u32 work_done;
788
789 work_done = ionic_cq_service(cq, cq->num_descs,
790 ionic_tx_service, NULL, NULL);
791 if (work_done)
792 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
793 work_done, IONIC_INTR_CRED_RESET_COALESCE);
794}
795
796void ionic_tx_empty(struct ionic_queue *q)
797{
798 struct ionic_desc_info *desc_info;
799 int bytes = 0;
800 int pkts = 0;
801
802
803 while (q->head_idx != q->tail_idx) {
804 desc_info = &q->info[q->tail_idx];
805 desc_info->bytes = 0;
806 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
807 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
808 if (desc_info->cb_arg) {
809 pkts++;
810 bytes += desc_info->bytes;
811 }
812 desc_info->cb = NULL;
813 desc_info->cb_arg = NULL;
814 }
815
816 if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
817 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
818}
819
820static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
821{
822 int err;
823
824 err = skb_cow_head(skb, 0);
825 if (err)
826 return err;
827
828 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
829 inner_ip_hdr(skb)->check = 0;
830 inner_tcp_hdr(skb)->check =
831 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
832 inner_ip_hdr(skb)->daddr,
833 0, IPPROTO_TCP, 0);
834 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
835 inner_tcp_hdr(skb)->check =
836 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
837 &inner_ipv6_hdr(skb)->daddr,
838 0, IPPROTO_TCP, 0);
839 }
840
841 return 0;
842}
843
844static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
845{
846 int err;
847
848 err = skb_cow_head(skb, 0);
849 if (err)
850 return err;
851
852 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
853 ip_hdr(skb)->check = 0;
854 tcp_hdr(skb)->check =
855 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
856 ip_hdr(skb)->daddr,
857 0, IPPROTO_TCP, 0);
858 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
859 tcp_v6_gso_csum_prep(skb);
860 }
861
862 return 0;
863}
864
865static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
866 struct sk_buff *skb,
867 dma_addr_t addr, u8 nsge, u16 len,
868 unsigned int hdrlen, unsigned int mss,
869 bool outer_csum,
870 u16 vlan_tci, bool has_vlan,
871 bool start, bool done)
872{
873 u8 flags = 0;
874 u64 cmd;
875
876 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
877 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
878 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
879 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
880
881 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
882 desc->cmd = cpu_to_le64(cmd);
883 desc->len = cpu_to_le16(len);
884 desc->vlan_tci = cpu_to_le16(vlan_tci);
885 desc->hdr_len = cpu_to_le16(hdrlen);
886 desc->mss = cpu_to_le16(mss);
887
888 if (start) {
889 skb_tx_timestamp(skb);
890 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
891 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
892 ionic_txq_post(q, false, ionic_tx_clean, skb);
893 } else {
894 ionic_txq_post(q, done, NULL, NULL);
895 }
896}
897
898static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
899{
900 struct ionic_tx_stats *stats = q_to_tx_stats(q);
901 struct ionic_desc_info *desc_info;
902 struct ionic_buf_info *buf_info;
903 struct ionic_txq_sg_elem *elem;
904 struct ionic_txq_desc *desc;
905 unsigned int chunk_len;
906 unsigned int frag_rem;
907 unsigned int tso_rem;
908 unsigned int seg_rem;
909 dma_addr_t desc_addr;
910 dma_addr_t frag_addr;
911 unsigned int hdrlen;
912 unsigned int len;
913 unsigned int mss;
914 bool start, done;
915 bool outer_csum;
916 bool has_vlan;
917 u16 desc_len;
918 u8 desc_nsge;
919 u16 vlan_tci;
920 bool encap;
921 int err;
922
923 desc_info = &q->info[q->head_idx];
924 buf_info = desc_info->bufs;
925
926 if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
927 return -EIO;
928
929 len = skb->len;
930 mss = skb_shinfo(skb)->gso_size;
931 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
932 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
933 has_vlan = !!skb_vlan_tag_present(skb);
934 vlan_tci = skb_vlan_tag_get(skb);
935 encap = skb->encapsulation;
936
937
938
939
940
941
942 if (encap)
943 err = ionic_tx_tcp_inner_pseudo_csum(skb);
944 else
945 err = ionic_tx_tcp_pseudo_csum(skb);
946 if (err)
947 return err;
948
949 if (encap)
950 hdrlen = skb_inner_transport_header(skb) - skb->data +
951 inner_tcp_hdrlen(skb);
952 else
953 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
954
955 tso_rem = len;
956 seg_rem = min(tso_rem, hdrlen + mss);
957
958 frag_addr = 0;
959 frag_rem = 0;
960
961 start = true;
962
963 while (tso_rem > 0) {
964 desc = NULL;
965 elem = NULL;
966 desc_addr = 0;
967 desc_len = 0;
968 desc_nsge = 0;
969
970 while (seg_rem > 0) {
971
972 if (frag_rem == 0) {
973
974 frag_addr = buf_info->dma_addr;
975 frag_rem = buf_info->len;
976 buf_info++;
977 }
978 chunk_len = min(frag_rem, seg_rem);
979 if (!desc) {
980
981 desc = desc_info->txq_desc;
982 elem = desc_info->txq_sg_desc->elems;
983 desc_addr = frag_addr;
984 desc_len = chunk_len;
985 } else {
986
987 elem->addr = cpu_to_le64(frag_addr);
988 elem->len = cpu_to_le16(chunk_len);
989 elem++;
990 desc_nsge++;
991 }
992 frag_addr += chunk_len;
993 frag_rem -= chunk_len;
994 tso_rem -= chunk_len;
995 seg_rem -= chunk_len;
996 }
997 seg_rem = min(tso_rem, mss);
998 done = (tso_rem == 0);
999
1000 ionic_tx_tso_post(q, desc, skb,
1001 desc_addr, desc_nsge, desc_len,
1002 hdrlen, mss, outer_csum, vlan_tci, has_vlan,
1003 start, done);
1004 start = false;
1005
1006 desc_info = &q->info[q->head_idx];
1007 desc_info->nbufs = 0;
1008 }
1009
1010 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
1011 stats->bytes += len;
1012 stats->tso++;
1013 stats->tso_bytes = len;
1014
1015 return 0;
1016}
1017
1018static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
1019 struct ionic_desc_info *desc_info)
1020{
1021 struct ionic_txq_desc *desc = desc_info->txq_desc;
1022 struct ionic_buf_info *buf_info = desc_info->bufs;
1023 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1024 bool has_vlan;
1025 u8 flags = 0;
1026 bool encap;
1027 u64 cmd;
1028
1029 has_vlan = !!skb_vlan_tag_present(skb);
1030 encap = skb->encapsulation;
1031
1032 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1033 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1034
1035 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
1036 flags, skb_shinfo(skb)->nr_frags,
1037 buf_info->dma_addr);
1038 desc->cmd = cpu_to_le64(cmd);
1039 desc->len = cpu_to_le16(buf_info->len);
1040 if (has_vlan) {
1041 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1042 stats->vlan_inserted++;
1043 } else {
1044 desc->vlan_tci = 0;
1045 }
1046 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
1047 desc->csum_offset = cpu_to_le16(skb->csum_offset);
1048
1049 if (skb_csum_is_sctp(skb))
1050 stats->crc32_csum++;
1051 else
1052 stats->csum++;
1053
1054 return 0;
1055}
1056
1057static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
1058 struct ionic_desc_info *desc_info)
1059{
1060 struct ionic_txq_desc *desc = desc_info->txq_desc;
1061 struct ionic_buf_info *buf_info = desc_info->bufs;
1062 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1063 bool has_vlan;
1064 u8 flags = 0;
1065 bool encap;
1066 u64 cmd;
1067
1068 has_vlan = !!skb_vlan_tag_present(skb);
1069 encap = skb->encapsulation;
1070
1071 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
1072 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
1073
1074 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
1075 flags, skb_shinfo(skb)->nr_frags,
1076 buf_info->dma_addr);
1077 desc->cmd = cpu_to_le64(cmd);
1078 desc->len = cpu_to_le16(buf_info->len);
1079 if (has_vlan) {
1080 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
1081 stats->vlan_inserted++;
1082 } else {
1083 desc->vlan_tci = 0;
1084 }
1085 desc->csum_start = 0;
1086 desc->csum_offset = 0;
1087
1088 stats->csum_none++;
1089
1090 return 0;
1091}
1092
1093static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
1094 struct ionic_desc_info *desc_info)
1095{
1096 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
1097 struct ionic_buf_info *buf_info = &desc_info->bufs[1];
1098 struct ionic_txq_sg_elem *elem = sg_desc->elems;
1099 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1100 unsigned int i;
1101
1102 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
1103 elem->addr = cpu_to_le64(buf_info->dma_addr);
1104 elem->len = cpu_to_le16(buf_info->len);
1105 }
1106
1107 stats->frags += skb_shinfo(skb)->nr_frags;
1108
1109 return 0;
1110}
1111
1112static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
1113{
1114 struct ionic_desc_info *desc_info = &q->info[q->head_idx];
1115 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1116 int err;
1117
1118 if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
1119 return -EIO;
1120
1121
1122 if (skb->ip_summed == CHECKSUM_PARTIAL)
1123 err = ionic_tx_calc_csum(q, skb, desc_info);
1124 else
1125 err = ionic_tx_calc_no_csum(q, skb, desc_info);
1126 if (err)
1127 return err;
1128
1129
1130 err = ionic_tx_skb_frags(q, skb, desc_info);
1131 if (err)
1132 return err;
1133
1134 skb_tx_timestamp(skb);
1135 stats->pkts++;
1136 stats->bytes += skb->len;
1137
1138 if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
1139 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
1140 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
1141
1142 return 0;
1143}
1144
1145static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
1146{
1147 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1148 int ndescs;
1149 int err;
1150
1151
1152 if (skb_is_gso(skb))
1153 ndescs = skb_shinfo(skb)->gso_segs;
1154 else
1155 ndescs = 1;
1156
1157
1158 if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems)
1159 return ndescs;
1160
1161
1162 err = skb_linearize(skb);
1163 if (err)
1164 return err;
1165
1166 stats->linearize++;
1167
1168 return ndescs;
1169}
1170
1171static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1172{
1173 int stopped = 0;
1174
1175 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1176 netif_stop_subqueue(q->lif->netdev, q->index);
1177 q->stop++;
1178 stopped = 1;
1179
1180
1181 smp_rmb();
1182 if (ionic_q_has_space(q, ndescs)) {
1183 netif_wake_subqueue(q->lif->netdev, q->index);
1184 stopped = 0;
1185 }
1186 }
1187
1188 return stopped;
1189}
1190
1191static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
1192 struct net_device *netdev)
1193{
1194 struct ionic_lif *lif = netdev_priv(netdev);
1195 struct ionic_queue *q = &lif->hwstamp_txq->q;
1196 int err, ndescs;
1197
1198
1199
1200
1201
1202
1203 ndescs = ionic_tx_descs_needed(q, skb);
1204 if (unlikely(ndescs < 0))
1205 goto err_out_drop;
1206
1207 if (unlikely(!ionic_q_has_space(q, ndescs)))
1208 goto err_out_drop;
1209
1210 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
1211 if (skb_is_gso(skb))
1212 err = ionic_tx_tso(q, skb);
1213 else
1214 err = ionic_tx(q, skb);
1215
1216 if (err)
1217 goto err_out_drop;
1218
1219 return NETDEV_TX_OK;
1220
1221err_out_drop:
1222 q->drop++;
1223 dev_kfree_skb(skb);
1224 return NETDEV_TX_OK;
1225}
1226
1227netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1228{
1229 u16 queue_index = skb_get_queue_mapping(skb);
1230 struct ionic_lif *lif = netdev_priv(netdev);
1231 struct ionic_queue *q;
1232 int ndescs;
1233 int err;
1234
1235 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
1236 dev_kfree_skb(skb);
1237 return NETDEV_TX_OK;
1238 }
1239
1240 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1241 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
1242 return ionic_start_hwstamp_xmit(skb, netdev);
1243
1244 if (unlikely(queue_index >= lif->nxqs))
1245 queue_index = 0;
1246 q = &lif->txqcqs[queue_index]->q;
1247
1248 ndescs = ionic_tx_descs_needed(q, skb);
1249 if (ndescs < 0)
1250 goto err_out_drop;
1251
1252 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1253 return NETDEV_TX_BUSY;
1254
1255 if (skb_is_gso(skb))
1256 err = ionic_tx_tso(q, skb);
1257 else
1258 err = ionic_tx(q, skb);
1259
1260 if (err)
1261 goto err_out_drop;
1262
1263
1264
1265
1266
1267 ionic_maybe_stop_tx(q, 4);
1268
1269 return NETDEV_TX_OK;
1270
1271err_out_drop:
1272 q->stop++;
1273 q->drop++;
1274 dev_kfree_skb(skb);
1275 return NETDEV_TX_OK;
1276}
1277
