1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/prefetch.h>
34#include <linux/ip.h>
35#include <linux/ipv6.h>
36#include <linux/tcp.h>
37#include <net/ip6_checksum.h>
38#include <net/page_pool.h>
39#include <net/inet_ecn.h>
40#include "en.h"
41#include "en/txrx.h"
42#include "en_tc.h"
43#include "eswitch.h"
44#include "en_rep.h"
45#include "en/rep/tc.h"
46#include "ipoib/ipoib.h"
47#include "accel/ipsec.h"
48#include "fpga/ipsec.h"
49#include "en_accel/ipsec_rxtx.h"
50#include "en_accel/tls_rxtx.h"
51#include "lib/clock.h"
52#include "en/xdp.h"
53#include "en/xsk/rx.h"
54#include "en/health.h"
55#include "en/params.h"
56#include "en/txrx.h"
57
58static struct sk_buff *
59mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
60 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
61static struct sk_buff *
62mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
63 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
64static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
65static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
66
67const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = {
68 .handle_rx_cqe = mlx5e_handle_rx_cqe,
69 .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
70};
71
72static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
73{
74 return config->rx_filter == HWTSTAMP_FILTER_ALL;
75}
76
77static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq,
78 u32 cqcc, void *data)
79{
80 u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
81
82 memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64));
83}
84
85static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
86 struct mlx5_cqwq *wq,
87 u32 cqcc)
88{
89 struct mlx5e_cq_decomp *cqd = &rq->cqd;
90 struct mlx5_cqe64 *title = &cqd->title;
91
92 mlx5e_read_cqe_slot(wq, cqcc, title);
93 cqd->left = be32_to_cpu(title->byte_cnt);
94 cqd->wqe_counter = be16_to_cpu(title->wqe_counter);
95 rq->stats->cqe_compress_blks++;
96}
97
98static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq *wq,
99 struct mlx5e_cq_decomp *cqd,
100 u32 cqcc)
101{
102 mlx5e_read_cqe_slot(wq, cqcc, cqd->mini_arr);
103 cqd->mini_arr_idx = 0;
104}
105
106static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq *wq, int n)
107{
108 u32 cqcc = wq->cc;
109 u8 op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;
110 u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
111 u32 wq_sz = mlx5_cqwq_get_size(wq);
112 u32 ci_top = min_t(u32, wq_sz, ci + n);
113
114 for (; ci < ci_top; ci++, n--) {
115 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
116
117 cqe->op_own = op_own;
118 }
119
120 if (unlikely(ci == wq_sz)) {
121 op_own = !op_own;
122 for (ci = 0; ci < n; ci++) {
123 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
124
125 cqe->op_own = op_own;
126 }
127 }
128}
129
130static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
131 struct mlx5_cqwq *wq,
132 u32 cqcc)
133{
134 struct mlx5e_cq_decomp *cqd = &rq->cqd;
135 struct mlx5_mini_cqe8 *mini_cqe = &cqd->mini_arr[cqd->mini_arr_idx];
136 struct mlx5_cqe64 *title = &cqd->title;
137
138 title->byte_cnt = mini_cqe->byte_cnt;
139 title->check_sum = mini_cqe->checksum;
140 title->op_own &= 0xf0;
141 title->op_own |= 0x01 & (cqcc >> wq->fbc.log_sz);
142 title->wqe_counter = cpu_to_be16(cqd->wqe_counter);
143
144 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
145 cqd->wqe_counter += mpwrq_get_cqe_consumed_strides(title);
146 else
147 cqd->wqe_counter =
148 mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cqd->wqe_counter + 1);
149}
150
151static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
152 struct mlx5_cqwq *wq,
153 u32 cqcc)
154{
155 struct mlx5e_cq_decomp *cqd = &rq->cqd;
156
157 mlx5e_decompress_cqe(rq, wq, cqcc);
158 cqd->title.rss_hash_type = 0;
159 cqd->title.rss_hash_result = 0;
160}
161
162static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
163 struct mlx5_cqwq *wq,
164 int update_owner_only,
165 int budget_rem)
166{
167 struct mlx5e_cq_decomp *cqd = &rq->cqd;
168 u32 cqcc = wq->cc + update_owner_only;
169 u32 cqe_count;
170 u32 i;
171
172 cqe_count = min_t(u32, cqd->left, budget_rem);
173
174 for (i = update_owner_only; i < cqe_count;
175 i++, cqd->mini_arr_idx++, cqcc++) {
176 if (cqd->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE)
177 mlx5e_read_mini_arr_slot(wq, cqd, cqcc);
178
179 mlx5e_decompress_cqe_no_hash(rq, wq, cqcc);
180 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
181 mlx5e_handle_rx_cqe, rq, &cqd->title);
182 }
183 mlx5e_cqes_update_owner(wq, cqcc - wq->cc);
184 wq->cc = cqcc;
185 cqd->left -= cqe_count;
186 rq->stats->cqe_compress_pkts += cqe_count;
187
188 return cqe_count;
189}
190
191static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
192 struct mlx5_cqwq *wq,
193 int budget_rem)
194{
195 struct mlx5e_cq_decomp *cqd = &rq->cqd;
196 u32 cc = wq->cc;
197
198 mlx5e_read_title_slot(rq, wq, cc);
199 mlx5e_read_mini_arr_slot(wq, cqd, cc + 1);
200 mlx5e_decompress_cqe(rq, wq, cc);
201 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
202 mlx5e_handle_rx_cqe, rq, &cqd->title);
203 cqd->mini_arr_idx++;
204
205 return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1;
206}
207
208static inline bool mlx5e_page_is_reserved(struct page *page)
209{
210 return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
211}
212
213static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
214 struct mlx5e_dma_info *dma_info)
215{
216 struct mlx5e_page_cache *cache = &rq->page_cache;
217 u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
218 struct mlx5e_rq_stats *stats = rq->stats;
219
220 if (tail_next == cache->head) {
221 stats->cache_full++;
222 return false;
223 }
224
225 if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
226 stats->cache_waive++;
227 return false;
228 }
229
230 cache->page_cache[cache->tail] = *dma_info;
231 cache->tail = tail_next;
232 return true;
233}
234
235static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
236 struct mlx5e_dma_info *dma_info)
237{
238 struct mlx5e_page_cache *cache = &rq->page_cache;
239 struct mlx5e_rq_stats *stats = rq->stats;
240
241 if (unlikely(cache->head == cache->tail)) {
242 stats->cache_empty++;
243 return false;
244 }
245
246 if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
247 stats->cache_busy++;
248 return false;
249 }
250
251 *dma_info = cache->page_cache[cache->head];
252 cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
253 stats->cache_reuse++;
254
255 dma_sync_single_for_device(rq->pdev, dma_info->addr,
256 PAGE_SIZE,
257 DMA_FROM_DEVICE);
258 return true;
259}
260
261static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
262 struct mlx5e_dma_info *dma_info)
263{
264 if (mlx5e_rx_cache_get(rq, dma_info))
265 return 0;
266
267 dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
268 if (unlikely(!dma_info->page))
269 return -ENOMEM;
270
271 dma_info->addr = dma_map_page(rq->pdev, dma_info->page, 0,
272 PAGE_SIZE, rq->buff.map_dir);
273 if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
274 page_pool_recycle_direct(rq->page_pool, dma_info->page);
275 dma_info->page = NULL;
276 return -ENOMEM;
277 }
278
279 return 0;
280}
281
282static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
283 struct mlx5e_dma_info *dma_info)
284{
285 if (rq->umem)
286 return mlx5e_xsk_page_alloc_umem(rq, dma_info);
287 else
288 return mlx5e_page_alloc_pool(rq, dma_info);
289}
290
291void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
292{
293 dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
294}
295
296void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
297 struct mlx5e_dma_info *dma_info,
298 bool recycle)
299{
300 if (likely(recycle)) {
301 if (mlx5e_rx_cache_put(rq, dma_info))
302 return;
303
304 mlx5e_page_dma_unmap(rq, dma_info);
305 page_pool_recycle_direct(rq->page_pool, dma_info->page);
306 } else {
307 mlx5e_page_dma_unmap(rq, dma_info);
308 page_pool_release_page(rq->page_pool, dma_info->page);
309 put_page(dma_info->page);
310 }
311}
312
313static inline void mlx5e_page_release(struct mlx5e_rq *rq,
314 struct mlx5e_dma_info *dma_info,
315 bool recycle)
316{
317 if (rq->umem)
318
319
320
321
322 xsk_buff_free(dma_info->xsk);
323 else
324 mlx5e_page_release_dynamic(rq, dma_info, recycle);
325}
326
327static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
328 struct mlx5e_wqe_frag_info *frag)
329{
330 int err = 0;
331
332 if (!frag->offset)
333
334
335
336
337
338 err = mlx5e_page_alloc(rq, frag->di);
339
340 return err;
341}
342
343static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
344 struct mlx5e_wqe_frag_info *frag,
345 bool recycle)
346{
347 if (frag->last_in_page)
348 mlx5e_page_release(rq, frag->di, recycle);
349}
350
351static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
352{
353 return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags];
354}
355
356static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
357 u16 ix)
358{
359 struct mlx5e_wqe_frag_info *frag = get_frag(rq, ix);
360 int err;
361 int i;
362
363 for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) {
364 err = mlx5e_get_rx_frag(rq, frag);
365 if (unlikely(err))
366 goto free_frags;
367
368 wqe->data[i].addr = cpu_to_be64(frag->di->addr +
369 frag->offset + rq->buff.headroom);
370 }
371
372 return 0;
373
374free_frags:
375 while (--i >= 0)
376 mlx5e_put_rx_frag(rq, --frag, true);
377
378 return err;
379}
380
381static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
382 struct mlx5e_wqe_frag_info *wi,
383 bool recycle)
384{
385 int i;
386
387 for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
388 mlx5e_put_rx_frag(rq, wi, recycle);
389}
390
391static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
392{
393 struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
394
395 mlx5e_free_rx_wqe(rq, wi, false);
396}
397
398static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
399{
400 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
401 int err;
402 int i;
403
404 if (rq->umem) {
405 int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
406
407
408
409
410
411 if (unlikely(!xsk_buff_can_alloc(rq->umem, pages_desired)))
412 return -ENOMEM;
413 }
414
415 for (i = 0; i < wqe_bulk; i++) {
416 struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
417
418 err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i);
419 if (unlikely(err))
420 goto free_wqes;
421 }
422
423 return 0;
424
425free_wqes:
426 while (--i >= 0)
427 mlx5e_dealloc_rx_wqe(rq, ix + i);
428
429 return err;
430}
431
432static inline void
433mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
434 struct mlx5e_dma_info *di, u32 frag_offset, u32 len,
435 unsigned int truesize)
436{
437 dma_sync_single_for_cpu(rq->pdev,
438 di->addr + frag_offset,
439 len, DMA_FROM_DEVICE);
440 page_ref_inc(di->page);
441 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
442 di->page, frag_offset, len, truesize);
443}
444
445static inline void
446mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
447 struct mlx5e_dma_info *dma_info,
448 int offset_from, u32 headlen)
449{
450 const void *from = page_address(dma_info->page) + offset_from;
451
452 unsigned int len = ALIGN(headlen, sizeof(long));
453
454 dma_sync_single_for_cpu(pdev, dma_info->addr + offset_from, len,
455 DMA_FROM_DEVICE);
456 skb_copy_to_linear_data(skb, from, len);
457}
458
459static void
460mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
461{
462 bool no_xdp_xmit;
463 struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
464 int i;
465
466
467 if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE))
468 return;
469
470 no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap,
471 MLX5_MPWRQ_PAGES_PER_WQE);
472
473 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
474 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
475 mlx5e_page_release(rq, &dma_info[i], recycle);
476}
477
478static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
479{
480 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
481
482 do {
483 u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head);
484
485 mlx5_wq_ll_push(wq, next_wqe_index);
486 } while (--n);
487
488
489 dma_wmb();
490
491 mlx5_wq_ll_update_db_record(wq);
492}
493
494static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
495{
496 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
497 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
498 struct mlx5e_icosq *sq = &rq->channel->icosq;
499 struct mlx5_wq_cyc *wq = &sq->wq;
500 struct mlx5e_umr_wqe *umr_wqe;
501 u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1);
502 u16 pi;
503 int err;
504 int i;
505
506
507
508
509 if (rq->umem &&
510 unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_PAGES_PER_WQE))) {
511 err = -ENOMEM;
512 goto err;
513 }
514
515 pi = mlx5e_icosq_get_next_pi(sq, MLX5E_UMR_WQEBBS);
516 umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
517 memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts));
518
519 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
520 err = mlx5e_page_alloc(rq, dma_info);
521 if (unlikely(err))
522 goto err_unmap;
523 umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
524 }
525
526 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
527 wi->consumed_strides = 0;
528
529 umr_wqe->ctrl.opmod_idx_opcode =
530 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
531 MLX5_OPCODE_UMR);
532 umr_wqe->uctrl.xlt_offset = cpu_to_be16(xlt_offset);
533
534 sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
535 .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
536 .num_wqebbs = MLX5E_UMR_WQEBBS,
537 .umr.rq = rq,
538 };
539
540 sq->pc += MLX5E_UMR_WQEBBS;
541
542 sq->doorbell_cseg = &umr_wqe->ctrl;
543
544 return 0;
545
546err_unmap:
547 while (--i >= 0) {
548 dma_info--;
549 mlx5e_page_release(rq, dma_info, true);
550 }
551
552err:
553 rq->stats->buff_alloc_err++;
554
555 return err;
556}
557
558static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
559{
560 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
561
562 mlx5e_free_rx_mpwqe(rq, wi, false);
563}
564
565INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
566{
567 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
568 u8 wqe_bulk;
569 int err;
570
571 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
572 return false;
573
574 wqe_bulk = rq->wqe.info.wqe_bulk;
575
576 if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
577 return false;
578
579 do {
580 u16 head = mlx5_wq_cyc_get_head(wq);
581
582 err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
583 if (unlikely(err)) {
584 rq->stats->buff_alloc_err++;
585 break;
586 }
587
588 mlx5_wq_cyc_push_n(wq, wqe_bulk);
589 } while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
590
591
592 dma_wmb();
593
594 mlx5_wq_cyc_update_db_record(wq);
595
596 return !!err;
597}
598
599void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq)
600{
601 u16 sqcc;
602
603 sqcc = sq->cc;
604
605 while (sqcc != sq->pc) {
606 struct mlx5e_icosq_wqe_info *wi;
607 u16 ci;
608
609 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
610 wi = &sq->db.wqe_info[ci];
611 sqcc += wi->num_wqebbs;
612#ifdef CONFIG_MLX5_EN_TLS
613 switch (wi->wqe_type) {
614 case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
615 mlx5e_ktls_handle_ctx_completion(wi);
616 break;
617 case MLX5E_ICOSQ_WQE_GET_PSV_TLS:
618 mlx5e_ktls_handle_get_psv_completion(wi, sq);
619 break;
620 }
621#endif
622 }
623 sq->cc = sqcc;
624}
625
626int mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
627{
628 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
629 struct mlx5_cqe64 *cqe;
630 u16 sqcc;
631 int i;
632
633 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
634 return 0;
635
636 cqe = mlx5_cqwq_get_cqe(&cq->wq);
637 if (likely(!cqe))
638 return 0;
639
640
641
642
643 sqcc = sq->cc;
644
645 i = 0;
646 do {
647 u16 wqe_counter;
648 bool last_wqe;
649
650 mlx5_cqwq_pop(&cq->wq);
651
652 wqe_counter = be16_to_cpu(cqe->wqe_counter);
653
654 do {
655 struct mlx5e_icosq_wqe_info *wi;
656 u16 ci;
657
658 last_wqe = (sqcc == wqe_counter);
659
660 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
661 wi = &sq->db.wqe_info[ci];
662 sqcc += wi->num_wqebbs;
663
664 if (last_wqe && unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
665 netdev_WARN_ONCE(cq->channel->netdev,
666 "Bad OP in ICOSQ CQE: 0x%x\n",
667 get_cqe_opcode(cqe));
668 mlx5e_dump_error_cqe(&sq->cq, sq->sqn,
669 (struct mlx5_err_cqe *)cqe);
670 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))
671 queue_work(cq->channel->priv->wq, &sq->recover_work);
672 break;
673 }
674
675 switch (wi->wqe_type) {
676 case MLX5E_ICOSQ_WQE_UMR_RX:
677 wi->umr.rq->mpwqe.umr_completed++;
678 break;
679 case MLX5E_ICOSQ_WQE_NOP:
680 break;
681#ifdef CONFIG_MLX5_EN_TLS
682 case MLX5E_ICOSQ_WQE_UMR_TLS:
683 break;
684 case MLX5E_ICOSQ_WQE_SET_PSV_TLS:
685 mlx5e_ktls_handle_ctx_completion(wi);
686 break;
687 case MLX5E_ICOSQ_WQE_GET_PSV_TLS:
688 mlx5e_ktls_handle_get_psv_completion(wi, sq);
689 break;
690#endif
691 default:
692 netdev_WARN_ONCE(cq->channel->netdev,
693 "Bad WQE type in ICOSQ WQE info: 0x%x\n",
694 wi->wqe_type);
695 }
696 } while (!last_wqe);
697 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
698
699 sq->cc = sqcc;
700
701 mlx5_cqwq_update_db_record(&cq->wq);
702
703 return i;
704}
705
706INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
707{
708 struct mlx5e_icosq *sq = &rq->channel->icosq;
709 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
710 u8 umr_completed = rq->mpwqe.umr_completed;
711 int alloc_err = 0;
712 u8 missing, i;
713 u16 head;
714
715 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
716 return false;
717
718 if (umr_completed) {
719 mlx5e_post_rx_mpwqe(rq, umr_completed);
720 rq->mpwqe.umr_in_progress -= umr_completed;
721 rq->mpwqe.umr_completed = 0;
722 }
723
724 missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
725
726 if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk))
727 rq->stats->congst_umr++;
728
729#define UMR_WQE_BULK (2)
730 if (likely(missing < UMR_WQE_BULK))
731 return false;
732
733 head = rq->mpwqe.actual_wq_head;
734 i = missing;
735 do {
736 alloc_err = mlx5e_alloc_rx_mpwqe(rq, head);
737
738 if (unlikely(alloc_err))
739 break;
740 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
741 } while (--i);
742
743 rq->mpwqe.umr_last_bulk = missing - i;
744 if (sq->doorbell_cseg) {
745 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
746 sq->doorbell_cseg = NULL;
747 }
748
749 rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
750 rq->mpwqe.actual_wq_head = head;
751
752
753
754
755
756
757
758 if (unlikely(alloc_err == -ENOMEM && rq->umem))
759 return true;
760
761 return false;
762}
763
764static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
765{
766 u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
767 u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
768 (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
769
770 tcp->check = 0;
771 tcp->psh = get_cqe_lro_tcppsh(cqe);
772
773 if (tcp_ack) {
774 tcp->ack = 1;
775 tcp->ack_seq = cqe->lro_ack_seq_num;
776 tcp->window = cqe->lro_tcp_win;
777 }
778}
779
780static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
781 u32 cqe_bcnt)
782{
783 struct ethhdr *eth = (struct ethhdr *)(skb->data);
784 struct tcphdr *tcp;
785 int network_depth = 0;
786 __wsum check;
787 __be16 proto;
788 u16 tot_len;
789 void *ip_p;
790
791 proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
792
793 tot_len = cqe_bcnt - network_depth;
794 ip_p = skb->data + network_depth;
795
796 if (proto == htons(ETH_P_IP)) {
797 struct iphdr *ipv4 = ip_p;
798
799 tcp = ip_p + sizeof(struct iphdr);
800 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
801
802 ipv4->ttl = cqe->lro_min_ttl;
803 ipv4->tot_len = cpu_to_be16(tot_len);
804 ipv4->check = 0;
805 ipv4->check = ip_fast_csum((unsigned char *)ipv4,
806 ipv4->ihl);
807
808 mlx5e_lro_update_tcp_hdr(cqe, tcp);
809 check = csum_partial(tcp, tcp->doff * 4,
810 csum_unfold((__force __sum16)cqe->check_sum));
811
812 tcp->check = csum_tcpudp_magic(ipv4->saddr, ipv4->daddr,
813 tot_len - sizeof(struct iphdr),
814 IPPROTO_TCP, check);
815 } else {
816 u16 payload_len = tot_len - sizeof(struct ipv6hdr);
817 struct ipv6hdr *ipv6 = ip_p;
818
819 tcp = ip_p + sizeof(struct ipv6hdr);
820 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
821
822 ipv6->hop_limit = cqe->lro_min_ttl;
823 ipv6->payload_len = cpu_to_be16(payload_len);
824
825 mlx5e_lro_update_tcp_hdr(cqe, tcp);
826 check = csum_partial(tcp, tcp->doff * 4,
827 csum_unfold((__force __sum16)cqe->check_sum));
828
829 tcp->check = csum_ipv6_magic(&ipv6->saddr, &ipv6->daddr, payload_len,
830 IPPROTO_TCP, check);
831 }
832}
833
834static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
835 struct sk_buff *skb)
836{
837 u8 cht = cqe->rss_hash_type;
838 int ht = (cht & CQE_RSS_HTYPE_L4) ? PKT_HASH_TYPE_L4 :
839 (cht & CQE_RSS_HTYPE_IP) ? PKT_HASH_TYPE_L3 :
840 PKT_HASH_TYPE_NONE;
841 skb_set_hash(skb, be32_to_cpu(cqe->rss_hash_result), ht);
842}
843
844static inline bool is_last_ethertype_ip(struct sk_buff *skb, int *network_depth,
845 __be16 *proto)
846{
847 *proto = ((struct ethhdr *)skb->data)->h_proto;
848 *proto = __vlan_get_protocol(skb, *proto, network_depth);
849
850 if (*proto == htons(ETH_P_IP))
851 return pskb_may_pull(skb, *network_depth + sizeof(struct iphdr));
852
853 if (*proto == htons(ETH_P_IPV6))
854 return pskb_may_pull(skb, *network_depth + sizeof(struct ipv6hdr));
855
856 return false;
857}
858
859static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb)
860{
861 int network_depth = 0;
862 __be16 proto;
863 void *ip;
864 int rc;
865
866 if (unlikely(!is_last_ethertype_ip(skb, &network_depth, &proto)))
867 return;
868
869 ip = skb->data + network_depth;
870 rc = ((proto == htons(ETH_P_IP)) ? IP_ECN_set_ce((struct iphdr *)ip) :
871 IP6_ECN_set_ce(skb, (struct ipv6hdr *)ip));
872
873 rq->stats->ecn_mark += !!rc;
874}
875
876static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto)
877{
878 void *ip_p = skb->data + network_depth;
879
880 return (proto == htons(ETH_P_IP)) ? ((struct iphdr *)ip_p)->protocol :
881 ((struct ipv6hdr *)ip_p)->nexthdr;
882}
883
884#define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
885
886#define MAX_PADDING 8
887
888static void
889tail_padding_csum_slow(struct sk_buff *skb, int offset, int len,
890 struct mlx5e_rq_stats *stats)
891{
892 stats->csum_complete_tail_slow++;
893 skb->csum = csum_block_add(skb->csum,
894 skb_checksum(skb, offset, len, 0),
895 offset);
896}
897
898static void
899tail_padding_csum(struct sk_buff *skb, int offset,
900 struct mlx5e_rq_stats *stats)
901{
902 u8 tail_padding[MAX_PADDING];
903 int len = skb->len - offset;
904 void *tail;
905
906 if (unlikely(len > MAX_PADDING)) {
907 tail_padding_csum_slow(skb, offset, len, stats);
908 return;
909 }
910
911 tail = skb_header_pointer(skb, offset, len, tail_padding);
912 if (unlikely(!tail)) {
913 tail_padding_csum_slow(skb, offset, len, stats);
914 return;
915 }
916
917 stats->csum_complete_tail++;
918 skb->csum = csum_block_add(skb->csum, csum_partial(tail, len, 0), offset);
919}
920
921static void
922mlx5e_skb_csum_fixup(struct sk_buff *skb, int network_depth, __be16 proto,
923 struct mlx5e_rq_stats *stats)
924{
925 struct ipv6hdr *ip6;
926 struct iphdr *ip4;
927 int pkt_len;
928
929
930 if (network_depth > ETH_HLEN)
931
932
933
934
935 skb->csum = csum_partial(skb->data + ETH_HLEN,
936 network_depth - ETH_HLEN,
937 skb->csum);
938
939
940 switch (proto) {
941 case htons(ETH_P_IP):
942 ip4 = (struct iphdr *)(skb->data + network_depth);
943 pkt_len = network_depth + ntohs(ip4->tot_len);
944 break;
945 case htons(ETH_P_IPV6):
946 ip6 = (struct ipv6hdr *)(skb->data + network_depth);
947 pkt_len = network_depth + sizeof(*ip6) + ntohs(ip6->payload_len);
948 break;
949 default:
950 return;
951 }
952
953 if (likely(pkt_len >= skb->len))
954 return;
955
956 tail_padding_csum(skb, pkt_len, stats);
957}
958
959static inline void mlx5e_handle_csum(struct net_device *netdev,
960 struct mlx5_cqe64 *cqe,
961 struct mlx5e_rq *rq,
962 struct sk_buff *skb,
963 bool lro)
964{
965 struct mlx5e_rq_stats *stats = rq->stats;
966 int network_depth = 0;
967 __be16 proto;
968
969 if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
970 goto csum_none;
971
972 if (lro) {
973 skb->ip_summed = CHECKSUM_UNNECESSARY;
974 stats->csum_unnecessary++;
975 return;
976 }
977
978
979 if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state))
980 goto csum_unnecessary;
981
982
983
984
985
986
987
988
989
990 if (short_frame(skb->len))
991 goto csum_unnecessary;
992
993 if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) {
994 u8 ipproto = get_ip_proto(skb, network_depth, proto);
995
996 if (unlikely(ipproto == IPPROTO_SCTP))
997 goto csum_unnecessary;
998
999 if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
1000 goto csum_none;
1001
1002 stats->csum_complete++;
1003 skb->ip_summed = CHECKSUM_COMPLETE;
1004 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
1005
1006 if (test_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state))
1007 return;
1008
1009
1010 mlx5e_skb_csum_fixup(skb, network_depth, proto, stats);
1011 return;
1012 }
1013
1014csum_unnecessary:
1015 if (likely((cqe->hds_ip_ext & CQE_L3_OK) &&
1016 (cqe->hds_ip_ext & CQE_L4_OK))) {
1017 skb->ip_summed = CHECKSUM_UNNECESSARY;
1018 if (cqe_is_tunneled(cqe)) {
1019 skb->csum_level = 1;
1020 skb->encapsulation = 1;
1021 stats->csum_unnecessary_inner++;
1022 return;
1023 }
1024 stats->csum_unnecessary++;
1025 return;
1026 }
1027csum_none:
1028 skb->ip_summed = CHECKSUM_NONE;
1029 stats->csum_none++;
1030}
1031
1032#define MLX5E_CE_BIT_MASK 0x80
1033
1034static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
1035 u32 cqe_bcnt,
1036 struct mlx5e_rq *rq,
1037 struct sk_buff *skb)
1038{
1039 u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
1040 struct mlx5e_rq_stats *stats = rq->stats;
1041 struct net_device *netdev = rq->netdev;
1042
1043 skb->mac_len = ETH_HLEN;
1044
1045 mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt);
1046
1047 if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
1048 mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);
1049
1050 if (lro_num_seg > 1) {
1051 mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
1052 skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
1053
1054
1055
1056 stats->packets += lro_num_seg - 1;
1057 stats->lro_packets++;
1058 stats->lro_bytes += cqe_bcnt;
1059 }
1060
1061 if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
1062 skb_hwtstamps(skb)->hwtstamp =
1063 mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));
1064
1065 skb_record_rx_queue(skb, rq->ix);
1066
1067 if (likely(netdev->features & NETIF_F_RXHASH))
1068 mlx5e_skb_set_hash(cqe, skb);
1069
1070 if (cqe_has_vlan(cqe)) {
1071 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1072 be16_to_cpu(cqe->vlan_info));
1073 stats->removed_vlan_packets++;
1074 }
1075
1076 skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
1077
1078 mlx5e_handle_csum(netdev, cqe, rq, skb, !!lro_num_seg);
1079
1080 if (unlikely(cqe->ml_path & MLX5E_CE_BIT_MASK))
1081 mlx5e_enable_ecn(rq, skb);
1082
1083 skb->protocol = eth_type_trans(skb, netdev);
1084
1085 if (unlikely(mlx5e_skb_is_multicast(skb)))
1086 stats->mcast_packets++;
1087}
1088
1089static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
1090 struct mlx5_cqe64 *cqe,
1091 u32 cqe_bcnt,
1092 struct sk_buff *skb)
1093{
1094 struct mlx5e_rq_stats *stats = rq->stats;
1095
1096 stats->packets++;
1097 stats->bytes += cqe_bcnt;
1098 mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
1099}
1100
1101static inline
1102struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
1103 u32 frag_size, u16 headroom,
1104 u32 cqe_bcnt)
1105{
1106 struct sk_buff *skb = build_skb(va, frag_size);
1107
1108 if (unlikely(!skb)) {
1109 rq->stats->buff_alloc_err++;
1110 return NULL;
1111 }
1112
1113 skb_reserve(skb, headroom);
1114 skb_put(skb, cqe_bcnt);
1115
1116 return skb;
1117}
1118
1119static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
1120 u32 len, struct xdp_buff *xdp)
1121{
1122 xdp->data_hard_start = va;
1123 xdp->data = va + headroom;
1124 xdp_set_data_meta_invalid(xdp);
1125 xdp->data_end = xdp->data + len;
1126 xdp->rxq = &rq->xdp_rxq;
1127 xdp->frame_sz = rq->buff.frame0_sz;
1128}
1129
1130static struct sk_buff *
1131mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1132 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1133{
1134 struct mlx5e_dma_info *di = wi->di;
1135 u16 rx_headroom = rq->buff.headroom;
1136 struct xdp_buff xdp;
1137 struct sk_buff *skb;
1138 void *va, *data;
1139 u32 frag_size;
1140
1141 va = page_address(di->page) + wi->offset;
1142 data = va + rx_headroom;
1143 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1144
1145 dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
1146 frag_size, DMA_FROM_DEVICE);
1147 prefetchw(va);
1148 prefetch(data);
1149
1150 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
1151 if (mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp))
1152 return NULL;
1153
1154 rx_headroom = xdp.data - xdp.data_hard_start;
1155 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1156 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
1157 if (unlikely(!skb))
1158 return NULL;
1159
1160
1161 page_ref_inc(di->page);
1162
1163 return skb;
1164}
1165
1166static struct sk_buff *
1167mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1168 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1169{
1170 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
1171 struct mlx5e_wqe_frag_info *head_wi = wi;
1172 u16 headlen = min_t(u32, MLX5E_RX_MAX_HEAD, cqe_bcnt);
1173 u16 frag_headlen = headlen;
1174 u16 byte_cnt = cqe_bcnt - headlen;
1175 struct sk_buff *skb;
1176
1177
1178
1179
1180 skb = napi_alloc_skb(rq->cq.napi,
1181 ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
1182 if (unlikely(!skb)) {
1183 rq->stats->buff_alloc_err++;
1184 return NULL;
1185 }
1186
1187 prefetchw(skb->data);
1188
1189 while (byte_cnt) {
1190 u16 frag_consumed_bytes =
1191 min_t(u16, frag_info->frag_size - frag_headlen, byte_cnt);
1192
1193 mlx5e_add_skb_frag(rq, skb, wi->di, wi->offset + frag_headlen,
1194 frag_consumed_bytes, frag_info->frag_stride);
1195 byte_cnt -= frag_consumed_bytes;
1196 frag_headlen = 0;
1197 frag_info++;
1198 wi++;
1199 }
1200
1201
1202 mlx5e_copy_skb_header(rq->pdev, skb, head_wi->di, head_wi->offset, headlen);
1203
1204 skb->tail += headlen;
1205 skb->len += headlen;
1206
1207 return skb;
1208}
1209
1210static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1211{
1212 struct mlx5_err_cqe *err_cqe = (struct mlx5_err_cqe *)cqe;
1213
1214 if (cqe_syndrome_needs_recover(err_cqe->syndrome) &&
1215 !test_and_set_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state)) {
1216 mlx5e_dump_error_cqe(&rq->cq, rq->rqn, err_cqe);
1217 queue_work(rq->channel->priv->wq, &rq->recover_work);
1218 }
1219}
1220
1221static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1222{
1223 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1224 struct mlx5e_wqe_frag_info *wi;
1225 struct sk_buff *skb;
1226 u32 cqe_bcnt;
1227 u16 ci;
1228
1229 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1230 wi = get_frag(rq, ci);
1231 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1232
1233 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1234 trigger_report(rq, cqe);
1235 rq->stats->wqe_err++;
1236 goto free_wqe;
1237 }
1238
1239 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1240 mlx5e_skb_from_cqe_linear,
1241 mlx5e_skb_from_cqe_nonlinear,
1242 rq, cqe, wi, cqe_bcnt);
1243 if (!skb) {
1244
1245 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1246
1247
1248
1249 goto wq_cyc_pop;
1250 }
1251 goto free_wqe;
1252 }
1253
1254 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1255 napi_gro_receive(rq->cq.napi, skb);
1256
1257free_wqe:
1258 mlx5e_free_rx_wqe(rq, wi, true);
1259wq_cyc_pop:
1260 mlx5_wq_cyc_pop(wq);
1261}
1262
1263#ifdef CONFIG_MLX5_ESWITCH
1264static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1265{
1266 struct net_device *netdev = rq->netdev;
1267 struct mlx5e_priv *priv = netdev_priv(netdev);
1268 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1269 struct mlx5_eswitch_rep *rep = rpriv->rep;
1270 struct mlx5e_tc_update_priv tc_priv = {};
1271 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1272 struct mlx5e_wqe_frag_info *wi;
1273 struct sk_buff *skb;
1274 u32 cqe_bcnt;
1275 u16 ci;
1276
1277 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1278 wi = get_frag(rq, ci);
1279 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1280
1281 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1282 rq->stats->wqe_err++;
1283 goto free_wqe;
1284 }
1285
1286 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1287 mlx5e_skb_from_cqe_linear,
1288 mlx5e_skb_from_cqe_nonlinear,
1289 rq, cqe, wi, cqe_bcnt);
1290 if (!skb) {
1291
1292 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1293
1294
1295
1296 goto wq_cyc_pop;
1297 }
1298 goto free_wqe;
1299 }
1300
1301 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1302
1303 if (rep->vlan && skb_vlan_tag_present(skb))
1304 skb_vlan_pop(skb);
1305
1306 if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))
1307 goto free_wqe;
1308
1309 napi_gro_receive(rq->cq.napi, skb);
1310
1311 mlx5_rep_tc_post_napi_receive(&tc_priv);
1312
1313free_wqe:
1314 mlx5e_free_rx_wqe(rq, wi, true);
1315wq_cyc_pop:
1316 mlx5_wq_cyc_pop(wq);
1317}
1318
1319static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1320{
1321 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
1322 u16 wqe_id = be16_to_cpu(cqe->wqe_id);
1323 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1324 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
1325 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
1326 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
1327 u32 page_idx = wqe_offset >> PAGE_SHIFT;
1328 struct mlx5e_tc_update_priv tc_priv = {};
1329 struct mlx5e_rx_wqe_ll *wqe;
1330 struct mlx5_wq_ll *wq;
1331 struct sk_buff *skb;
1332 u16 cqe_bcnt;
1333
1334 wi->consumed_strides += cstrides;
1335
1336 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1337 trigger_report(rq, cqe);
1338 rq->stats->wqe_err++;
1339 goto mpwrq_cqe_out;
1340 }
1341
1342 if (unlikely(mpwrq_is_filler_cqe(cqe))) {
1343 struct mlx5e_rq_stats *stats = rq->stats;
1344
1345 stats->mpwqe_filler_cqes++;
1346 stats->mpwqe_filler_strides += cstrides;
1347 goto mpwrq_cqe_out;
1348 }
1349
1350 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1351
1352 skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
1353 mlx5e_skb_from_cqe_mpwrq_linear,
1354 mlx5e_skb_from_cqe_mpwrq_nonlinear,
1355 rq, wi, cqe_bcnt, head_offset, page_idx);
1356 if (!skb)
1357 goto mpwrq_cqe_out;
1358
1359 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1360
1361 if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))
1362 goto mpwrq_cqe_out;
1363
1364 napi_gro_receive(rq->cq.napi, skb);
1365
1366 mlx5_rep_tc_post_napi_receive(&tc_priv);
1367
1368mpwrq_cqe_out:
1369 if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1370 return;
1371
1372 wq = &rq->mpwqe.wq;
1373 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1374 mlx5e_free_rx_mpwqe(rq, wi, true);
1375 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1376}
1377
1378const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = {
1379 .handle_rx_cqe = mlx5e_handle_rx_cqe_rep,
1380 .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep,
1381};
1382#endif
1383
1384static struct sk_buff *
1385mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1386 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1387{
1388 u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
1389 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1390 u32 frag_offset = head_offset + headlen;
1391 u32 byte_cnt = cqe_bcnt - headlen;
1392 struct mlx5e_dma_info *head_di = di;
1393 struct sk_buff *skb;
1394
1395 skb = napi_alloc_skb(rq->cq.napi,
1396 ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
1397 if (unlikely(!skb)) {
1398 rq->stats->buff_alloc_err++;
1399 return NULL;
1400 }
1401
1402 prefetchw(skb->data);
1403
1404 if (unlikely(frag_offset >= PAGE_SIZE)) {
1405 di++;
1406 frag_offset -= PAGE_SIZE;
1407 }
1408
1409 while (byte_cnt) {
1410 u32 pg_consumed_bytes =
1411 min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
1412 unsigned int truesize =
1413 ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
1414
1415 mlx5e_add_skb_frag(rq, skb, di, frag_offset,
1416 pg_consumed_bytes, truesize);
1417 byte_cnt -= pg_consumed_bytes;
1418 frag_offset = 0;
1419 di++;
1420 }
1421
1422 mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, headlen);
1423
1424 skb->tail += headlen;
1425 skb->len += headlen;
1426
1427 return skb;
1428}
1429
1430static struct sk_buff *
1431mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1432 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1433{
1434 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1435 u16 rx_headroom = rq->buff.headroom;
1436 u32 cqe_bcnt32 = cqe_bcnt;
1437 struct xdp_buff xdp;
1438 struct sk_buff *skb;
1439 void *va, *data;
1440 u32 frag_size;
1441
1442
1443 if (unlikely(cqe_bcnt > rq->hw_mtu)) {
1444 rq->stats->oversize_pkts_sw_drop++;
1445 return NULL;
1446 }
1447
1448 va = page_address(di->page) + head_offset;
1449 data = va + rx_headroom;
1450 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1451
1452 dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1453 frag_size, DMA_FROM_DEVICE);
1454 prefetchw(va);
1455 prefetch(data);
1456
1457 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
1458 if (mlx5e_xdp_handle(rq, di, &cqe_bcnt32, &xdp)) {
1459 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
1460 __set_bit(page_idx, wi->xdp_xmit_bitmap);
1461 return NULL;
1462 }
1463
1464 rx_headroom = xdp.data - xdp.data_hard_start;
1465 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1466 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
1467 if (unlikely(!skb))
1468 return NULL;
1469
1470
1471 page_ref_inc(di->page);
1472
1473 return skb;
1474}
1475
1476static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1477{
1478 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
1479 u16 wqe_id = be16_to_cpu(cqe->wqe_id);
1480 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1481 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
1482 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
1483 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
1484 u32 page_idx = wqe_offset >> PAGE_SHIFT;
1485 struct mlx5e_rx_wqe_ll *wqe;
1486 struct mlx5_wq_ll *wq;
1487 struct sk_buff *skb;
1488 u16 cqe_bcnt;
1489
1490 wi->consumed_strides += cstrides;
1491
1492 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1493 trigger_report(rq, cqe);
1494 rq->stats->wqe_err++;
1495 goto mpwrq_cqe_out;
1496 }
1497
1498 if (unlikely(mpwrq_is_filler_cqe(cqe))) {
1499 struct mlx5e_rq_stats *stats = rq->stats;
1500
1501 stats->mpwqe_filler_cqes++;
1502 stats->mpwqe_filler_strides += cstrides;
1503 goto mpwrq_cqe_out;
1504 }
1505
1506 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1507
1508 skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
1509 mlx5e_skb_from_cqe_mpwrq_linear,
1510 mlx5e_skb_from_cqe_mpwrq_nonlinear,
1511 rq, wi, cqe_bcnt, head_offset, page_idx);
1512 if (!skb)
1513 goto mpwrq_cqe_out;
1514
1515 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1516 napi_gro_receive(rq->cq.napi, skb);
1517
1518mpwrq_cqe_out:
1519 if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1520 return;
1521
1522 wq = &rq->mpwqe.wq;
1523 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1524 mlx5e_free_rx_mpwqe(rq, wi, true);
1525 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1526}
1527
1528int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
1529{
1530 struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
1531 struct mlx5_cqwq *cqwq = &cq->wq;
1532 struct mlx5_cqe64 *cqe;
1533 int work_done = 0;
1534
1535 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
1536 return 0;
1537
1538 if (rq->page_pool)
1539 page_pool_nid_changed(rq->page_pool, numa_mem_id());
1540
1541 if (rq->cqd.left) {
1542 work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget);
1543 if (rq->cqd.left || work_done >= budget)
1544 goto out;
1545 }
1546
1547 cqe = mlx5_cqwq_get_cqe(cqwq);
1548 if (!cqe) {
1549 if (unlikely(work_done))
1550 goto out;
1551 return 0;
1552 }
1553
1554 do {
1555 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
1556 work_done +=
1557 mlx5e_decompress_cqes_start(rq, cqwq,
1558 budget - work_done);
1559 continue;
1560 }
1561
1562 mlx5_cqwq_pop(cqwq);
1563
1564 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
1565 mlx5e_handle_rx_cqe, rq, cqe);
1566 } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
1567
1568out:
1569 if (rq->xdp_prog)
1570 mlx5e_xdp_rx_poll_complete(rq);
1571
1572 mlx5_cqwq_update_db_record(cqwq);
1573
1574
1575 wmb();
1576
1577 return work_done;
1578}
1579
1580#ifdef CONFIG_MLX5_CORE_IPOIB
1581
1582#define MLX5_IB_GRH_SGID_OFFSET 8
1583#define MLX5_IB_GRH_DGID_OFFSET 24
1584#define MLX5_GID_SIZE 16
1585
1586static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
1587 struct mlx5_cqe64 *cqe,
1588 u32 cqe_bcnt,
1589 struct sk_buff *skb)
1590{
1591 struct hwtstamp_config *tstamp;
1592 struct mlx5e_rq_stats *stats;
1593 struct net_device *netdev;
1594 struct mlx5e_priv *priv;
1595 char *pseudo_header;
1596 u32 flags_rqpn;
1597 u32 qpn;
1598 u8 *dgid;
1599 u8 g;
1600
1601 qpn = be32_to_cpu(cqe->sop_drop_qpn) & 0xffffff;
1602 netdev = mlx5i_pkey_get_netdev(rq->netdev, qpn);
1603
1604
1605
1606
1607 if (unlikely(!netdev)) {
1608
1609 skb->dev = NULL;
1610 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn);
1611 return;
1612 }
1613
1614 priv = mlx5i_epriv(netdev);
1615 tstamp = &priv->tstamp;
1616 stats = &priv->channel_stats[rq->ix].rq;
1617
1618 flags_rqpn = be32_to_cpu(cqe->flags_rqpn);
1619 g = (flags_rqpn >> 28) & 3;
1620 dgid = skb->data + MLX5_IB_GRH_DGID_OFFSET;
1621 if ((!g) || dgid[0] != 0xff)
1622 skb->pkt_type = PACKET_HOST;
1623 else if (memcmp(dgid, netdev->broadcast + 4, MLX5_GID_SIZE) == 0)
1624 skb->pkt_type = PACKET_BROADCAST;
1625 else
1626 skb->pkt_type = PACKET_MULTICAST;
1627
1628
1629
1630
1631 if (g && (qpn == (flags_rqpn & 0xffffff)) &&
1632 (memcmp(netdev->dev_addr + 4, skb->data + MLX5_IB_GRH_SGID_OFFSET,
1633 MLX5_GID_SIZE) == 0)) {
1634 skb->dev = NULL;
1635 return;
1636 }
1637
1638 skb_pull(skb, MLX5_IB_GRH_BYTES);
1639
1640 skb->protocol = *((__be16 *)(skb->data));
1641
1642 if (netdev->features & NETIF_F_RXCSUM) {
1643 skb->ip_summed = CHECKSUM_COMPLETE;
1644 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
1645 stats->csum_complete++;
1646 } else {
1647 skb->ip_summed = CHECKSUM_NONE;
1648 stats->csum_none++;
1649 }
1650
1651 if (unlikely(mlx5e_rx_hw_stamp(tstamp)))
1652 skb_hwtstamps(skb)->hwtstamp =
1653 mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));
1654
1655 skb_record_rx_queue(skb, rq->ix);
1656
1657 if (likely(netdev->features & NETIF_F_RXHASH))
1658 mlx5e_skb_set_hash(cqe, skb);
1659
1660
1661 pseudo_header = skb_push(skb, MLX5_IPOIB_PSEUDO_LEN);
1662 memset(pseudo_header, 0, MLX5_IPOIB_PSEUDO_LEN);
1663 skb_reset_mac_header(skb);
1664 skb_pull(skb, MLX5_IPOIB_HARD_LEN);
1665
1666 skb->dev = netdev;
1667
1668 stats->packets++;
1669 stats->bytes += cqe_bcnt;
1670}
1671
1672static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1673{
1674 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1675 struct mlx5e_wqe_frag_info *wi;
1676 struct sk_buff *skb;
1677 u32 cqe_bcnt;
1678 u16 ci;
1679
1680 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1681 wi = get_frag(rq, ci);
1682 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1683
1684 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1685 rq->stats->wqe_err++;
1686 goto wq_free_wqe;
1687 }
1688
1689 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1690 mlx5e_skb_from_cqe_linear,
1691 mlx5e_skb_from_cqe_nonlinear,
1692 rq, cqe, wi, cqe_bcnt);
1693 if (!skb)
1694 goto wq_free_wqe;
1695
1696 mlx5i_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1697 if (unlikely(!skb->dev)) {
1698 dev_kfree_skb_any(skb);
1699 goto wq_free_wqe;
1700 }
1701 napi_gro_receive(rq->cq.napi, skb);
1702
1703wq_free_wqe:
1704 mlx5e_free_rx_wqe(rq, wi, true);
1705 mlx5_wq_cyc_pop(wq);
1706}
1707
1708const struct mlx5e_rx_handlers mlx5i_rx_handlers = {
1709 .handle_rx_cqe = mlx5i_handle_rx_cqe,
1710 .handle_rx_cqe_mpwqe = NULL,
1711};
1712#endif
1713
1714#ifdef CONFIG_MLX5_EN_IPSEC
1715
1716static void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1717{
1718 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1719 struct mlx5e_wqe_frag_info *wi;
1720 struct sk_buff *skb;
1721 u32 cqe_bcnt;
1722 u16 ci;
1723
1724 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1725 wi = get_frag(rq, ci);
1726 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1727
1728 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1729 rq->stats->wqe_err++;
1730 goto wq_free_wqe;
1731 }
1732
1733 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1734 mlx5e_skb_from_cqe_linear,
1735 mlx5e_skb_from_cqe_nonlinear,
1736 rq, cqe, wi, cqe_bcnt);
1737 if (unlikely(!skb))
1738 goto wq_free_wqe;
1739
1740 skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
1741 if (unlikely(!skb))
1742 goto wq_free_wqe;
1743
1744 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1745 napi_gro_receive(rq->cq.napi, skb);
1746
1747wq_free_wqe:
1748 mlx5e_free_rx_wqe(rq, wi, true);
1749 mlx5_wq_cyc_pop(wq);
1750}
1751
1752#endif
1753
1754int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk)
1755{
1756 struct mlx5_core_dev *mdev = rq->mdev;
1757 struct mlx5e_channel *c = rq->channel;
1758
1759 switch (rq->wq_type) {
1760 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1761 rq->mpwqe.skb_from_cqe_mpwrq = xsk ?
1762 mlx5e_xsk_skb_from_cqe_mpwrq_linear :
1763 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ?
1764 mlx5e_skb_from_cqe_mpwrq_linear :
1765 mlx5e_skb_from_cqe_mpwrq_nonlinear;
1766 rq->post_wqes = mlx5e_post_rx_mpwqes;
1767 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
1768
1769 rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe_mpwqe;
1770#ifdef CONFIG_MLX5_EN_IPSEC
1771 if (MLX5_IPSEC_DEV(mdev)) {
1772 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
1773 return -EINVAL;
1774 }
1775#endif
1776 if (!rq->handle_rx_cqe) {
1777 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set\n");
1778 return -EINVAL;
1779 }
1780 break;
1781 default:
1782 rq->wqe.skb_from_cqe = xsk ?
1783 mlx5e_xsk_skb_from_cqe_linear :
1784 mlx5e_rx_is_linear_skb(params, NULL) ?
1785 mlx5e_skb_from_cqe_linear :
1786 mlx5e_skb_from_cqe_nonlinear;
1787 rq->post_wqes = mlx5e_post_rx_wqes;
1788 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
1789
1790#ifdef CONFIG_MLX5_EN_IPSEC
1791 if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
1792 c->priv->ipsec)
1793 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
1794 else
1795#endif
1796 rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe;
1797 if (!rq->handle_rx_cqe) {
1798 netdev_err(c->netdev, "RX handler of RQ is not set\n");
1799 return -EINVAL;
1800 }
1801 }
1802
1803 return 0;
1804}
1805